Skip to content

Commit

Permalink
feat: optionally add a byte order mark to csv export (#2455)
Browse files Browse the repository at this point in the history
rikuke authored Nov 16, 2023
1 parent d596162 commit 1d23a7a
Showing 3 changed files with 14 additions and 6 deletions.
Original file line number Diff line number Diff line change
@@ -190,7 +190,7 @@ def talpa_export_batch(self, request, *args, **kwargs) -> StreamingHttpResponse:
)

response = StreamingHttpResponse(
csv_service.get_csv_string_lines_generator(True),
csv_service.get_csv_string_lines_generator(True, True),
content_type="text/csv",
)
response["Content-Disposition"] = "attachment; filename={filename}.csv".format(
8 changes: 7 additions & 1 deletion backend/benefit/applications/services/csv_export_base.py
Original file line number Diff line number Diff line change
@@ -117,19 +117,25 @@ def get_csv_cell_list_lines_generator(
yield line

def get_csv_string_lines_generator(
self, remove_quotes: bool = False
self, remove_quotes: bool = False, add_bom: bool = False
) -> Generator[str, None, None]:
"""
Generate CSV's string lines using self.get_csv_cell_list_lines_generator().
:return: Generator which generates list of strings that each end with '\r\n'.
Passing remove_quotes=True will disable quoting of values as it is required by the Talpa integration.
Passing add_bom=True will add a BOM (Byte Order Mark) at the beginning of the file.
"""
quoting = csv.QUOTE_NONE if remove_quotes else csv.QUOTE_NONNUMERIC

io = StringIO()
csv_writer = csv.writer(io, delimiter=self.CSV_DELIMITER, quoting=quoting)
line_length_set = set()

# Add BOM as the first item in the generator
if add_bom:
yield "\ufeff"

for line in self.get_csv_cell_list_lines_generator():
line_length_set.add(len(line))
assert len(line_length_set) == 1, "Each CSV line must have same colum count"
10 changes: 6 additions & 4 deletions backend/benefit/applications/tests/test_talpa_integration.py
Original file line number Diff line number Diff line change
@@ -40,10 +40,12 @@ def test_talpa_csv_output(pruned_applications_csv_service_with_one_application):
csv_lines = split_lines_at_semicolon(
pruned_applications_csv_service_with_one_application.get_csv_string()
)
# BOM at the beginning of the file
assert csv_lines[0][0] == '"Hakemusnumero"'
for idx, col in enumerate(
pruned_applications_csv_service_with_one_application.CSV_COLUMNS
):
csv_columns = iter(pruned_applications_csv_service_with_one_application.CSV_COLUMNS)
next(csv_columns, None) # Skip the first element

for idx, col in enumerate(csv_columns, start=1):
assert csv_lines[0][idx] == f'"{col.heading}"'

assert (
@@ -112,7 +114,7 @@ def test_write_talpa_csv_file(
application.save()
output_file = tmp_path / "output.csv"
pruned_applications_csv_service_with_one_application.write_csv_file(output_file)
with open(output_file, encoding="utf-8") as f:
with open(output_file, encoding="utf-8-sig") as f:
contents = f.read()
assert contents.startswith('"Hakemusnumero";"Työnantajan tyyppi"')
assert "äöÄÖtest" in contents

0 comments on commit 1d23a7a

Please sign in to comment.