mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-10-24 10:17:05 +02:00
fix: auto-detect single-column CSV files
Format detection passed the first bytes to csv.Sniffer().sniff(), which
raises csv.Error('Could not determine delimiter') for files with no
delimiter at all, so a bare single-column list like 'id\n1\n2' could
never be loaded through rows_from_file. Fall back to the default CSV
dialect when sniffing fails so DictReader treats the file as a single
column.
This commit is contained in:
parent
6bc1d33d58
commit
04c67240ac
2 changed files with 29 additions and 4 deletions
|
|
@ -389,14 +389,24 @@ def rows_from_file(
|
||||||
with buffered:
|
with buffered:
|
||||||
return rows_from_file(buffered, format=Format.JSON)
|
return rows_from_file(buffered, format=Format.JSON)
|
||||||
else:
|
else:
|
||||||
dialect = csv.Sniffer().sniff(
|
try:
|
||||||
first_bytes.decode(encoding or "utf-8-sig", "ignore")
|
dialect = csv.Sniffer().sniff(
|
||||||
)
|
first_bytes.decode(encoding or "utf-8-sig", "ignore")
|
||||||
|
)
|
||||||
|
except csv.Error:
|
||||||
|
# Single-column files have no delimiter to sniff (e.g. a bare
|
||||||
|
# "id\n1\n2\n" list). Fall back to the default CSV dialect so
|
||||||
|
# DictReader still loads them as a single column.
|
||||||
|
dialect = None
|
||||||
rows, _ = rows_from_file(
|
rows, _ = rows_from_file(
|
||||||
buffered, format=Format.CSV, dialect=dialect, encoding=encoding
|
buffered, format=Format.CSV, dialect=dialect, encoding=encoding
|
||||||
)
|
)
|
||||||
# Make sure we return the format we detected
|
# Make sure we return the format we detected
|
||||||
detected_format = Format.TSV if dialect.delimiter == "\t" else Format.CSV
|
detected_format = (
|
||||||
|
Format.TSV
|
||||||
|
if dialect is not None and dialect.delimiter == "\t"
|
||||||
|
else Format.CSV
|
||||||
|
)
|
||||||
return (
|
return (
|
||||||
_extra_key_strategy(
|
_extra_key_strategy(
|
||||||
cast(Iterable[dict[str | None, object]], rows),
|
cast(Iterable[dict[str | None, object]], rows),
|
||||||
|
|
|
||||||
|
|
@ -55,6 +55,21 @@ def test_hash_record():
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"content,expected",
|
||||||
|
(
|
||||||
|
(b"apple\nbanana\ncherry\n", [{"apple": "banana"}, {"apple": "cherry"}]),
|
||||||
|
(b"id\n1\n2\n", [{"id": "1"}, {"id": "2"}]),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
def test_rows_from_file_detects_single_column_csv(content, expected):
|
||||||
|
# A single-column file has no delimiter for csv.Sniffer to detect; it
|
||||||
|
# used to raise csv.Error("Could not determine delimiter").
|
||||||
|
rows, format_ = utils.rows_from_file(io.BytesIO(content))
|
||||||
|
assert format_ == utils.Format.CSV
|
||||||
|
assert list(rows) == expected
|
||||||
|
|
||||||
|
|
||||||
def test_maximize_csv_field_size_limit():
|
def test_maximize_csv_field_size_limit():
|
||||||
# Reset to default in case other tests have changed it
|
# Reset to default in case other tests have changed it
|
||||||
csv.field_size_limit(utils.ORIGINAL_CSV_FIELD_SIZE_LIMIT)
|
csv.field_size_limit(utils.ORIGINAL_CSV_FIELD_SIZE_LIMIT)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue