Preserve carriage returns in quoted CSV/TSV fields (#895)

When reading CSV or TSV files through rows_from_file() or the CLI insert command, io.TextIOWrapper defaulted to universal-newline mode (newline=None), causing carriage returns (\r\n or lone \r) embedded inside quoted fields to be silently translated to \n before csv.reader could process them.

Setting newline='' in TextIOWrapper preserves carriage returns in quoted fields according to the Python standard library csv module specification.

Fixes #895
This commit is contained in:
agustin18 2026-10-03 23:08:20 +00:00
commit c27092220d
4 changed files with 49 additions and 3 deletions

View file

@ -131,3 +131,31 @@ def test_detect_format_keeps_streaming_reader_open(
assert not buffered_readers[0].closed
assert len(list(rows)) == 2000
assert buffered_readers[0].closed
@pytest.mark.parametrize(
"file_bytes,expected_format,expected_rows",
[
(
b'id,v\r\n1,"line1\r\nline2"\r\n',
Format.CSV,
[{"id": "1", "v": "line1\r\nline2"}],
),
(
b'id\tv\r\n1\t"tab1\r\ntab2"\r\n',
Format.TSV,
[{"id": "1", "v": "tab1\r\ntab2"}],
),
(
b'id,v\r\n1,"isolated\rreturn"\r\n',
Format.CSV,
[{"id": "1", "v": "isolated\rreturn"}],
),
],
)
def test_rows_from_file_preserves_carriage_returns_in_quoted_fields(
file_bytes, expected_format, expected_rows
):
rows, detected_format = rows_from_file(BytesIO(file_bytes), format=expected_format)
assert detected_format == expected_format
assert list(rows) == expected_rows