remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-10-27 21:57:11 +01:00
Code Issues Projects Releases Packages Wiki Activity

Fix delimiter detection in rows_from_file() for ragged and single-column files

Browse source 
When a CSV or TSV file was not perfectly rectangular, `csv.Sniffer` either
raised "Could not determine delimiter" (single-column files) or returned a
nonsense delimiter such as "a" for "id,name\n1,Cleo,extra". The latter
silently turned the header into a single column called "id,n" and broke the
documented extra-field handling.

Treat an alphanumeric sniffed delimiter as a failed sniff and fall back to
whichever of comma or tab appears on the most lines, using comma when
neither appears (a single-column file). Well-formed files are unaffected.
This commit is contained in:
makiaveli1 2026-09-29 06:23:27 +01:00
commit 6da8638e18
No known key found for this signature in database
2 changed files with 74 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

32
sqlite_utils/utils.py
View file

@ -285,6 +285,20 @@ def _extra_key_strategy(
yield row_out
def _fallback_dialect(sample: str) -> type[csv.Dialect]:
"""Return comma or tab for a sample that could not be sniffed.
Whichever appears on the most non-blank lines wins. Comma is used when the
sample contains neither, which is what a single-column file looks like.
"""
lines = [line for line in sample.splitlines() if line.strip()]
if not lines:
return csv.excel
commas = sum(1 for line in lines if "," in line)
tabs = sum(1 for line in lines if "\t" in line)
return csv.excel_tab if tabs > commas else csv.excel
def rows_from_file(
fp: BinaryIO,
format: Format | None = None,
@ -389,9 +403,21 @@ def rows_from_file(
with buffered:
return rows_from_file(buffered, format=Format.JSON)
else:
dialect = csv.Sniffer().sniff(
first_bytes.decode(encoding or "utf-8-sig", "ignore")
)
sample = first_bytes.decode(encoding or "utf-8-sig", "ignore")
try:
dialect = csv.Sniffer().sniff(sample)
except csv.Error:
# Sniffing needs a consistent structure, so it fails on
# single-column and on ragged files.
dialect = _fallback_dialect(sample)
else:
# For ragged input the sniffer does not fail, it returns a
# nonsense delimiter: it reports "a" as the delimiter of
# "id,name\n1,Cleo,extra", which silently mangles the column
# names. A delimiter is punctuation, never a letter or digit, so
# treat an alphanumeric answer as a failed sniff.
if dialect.delimiter.isalnum():
dialect = _fallback_dialect(sample)
rows, _ = rows_from_file(
buffered, format=Format.CSV, dialect=dialect, encoding=encoding
)