mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-10-24 18:27:01 +02:00
Preserve carriage returns in quoted CSV/TSV fields (#895)
When reading CSV or TSV files through rows_from_file() or the CLI insert command, io.TextIOWrapper defaulted to universal-newline mode (newline=None), causing carriage returns (\r\n or lone \r) embedded inside quoted fields to be silently translated to \n before csv.reader could process them. Setting newline='' in TextIOWrapper preserves carriage returns in quoted fields according to the Python standard library csv module specification. Fixes #895
This commit is contained in:
parent
6bc1d33d58
commit
c27092220d
4 changed files with 49 additions and 3 deletions
|
|
@ -131,3 +131,31 @@ def test_detect_format_keeps_streaming_reader_open(
|
|||
assert not buffered_readers[0].closed
|
||||
assert len(list(rows)) == 2000
|
||||
assert buffered_readers[0].closed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"file_bytes,expected_format,expected_rows",
|
||||
[
|
||||
(
|
||||
b'id,v\r\n1,"line1\r\nline2"\r\n',
|
||||
Format.CSV,
|
||||
[{"id": "1", "v": "line1\r\nline2"}],
|
||||
),
|
||||
(
|
||||
b'id\tv\r\n1\t"tab1\r\ntab2"\r\n',
|
||||
Format.TSV,
|
||||
[{"id": "1", "v": "tab1\r\ntab2"}],
|
||||
),
|
||||
(
|
||||
b'id,v\r\n1,"isolated\rreturn"\r\n',
|
||||
Format.CSV,
|
||||
[{"id": "1", "v": "isolated\rreturn"}],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_rows_from_file_preserves_carriage_returns_in_quoted_fields(
|
||||
file_bytes, expected_format, expected_rows
|
||||
):
|
||||
rows, detected_format = rows_from_file(BytesIO(file_bytes), format=expected_format)
|
||||
assert detected_format == expected_format
|
||||
assert list(rows) == expected_rows
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue