mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-10-25 02:37:07 +02:00
Read a single-column CSV file as one column when detecting the format
rows_from_file() sniffed the CSV dialect with no delimiters= list, so
csv.Sniffer considered every character. A single-column file then either
had no consistent candidate (csv.Error: Could not determine delimiter) or
was split on a letter that repeats on each line: a column of URLs came out
as {"ur": "https://examp", "": "e.com/a"}.
Format detection now sniffs for a comma, a semicolon, a tab or a pipe,
and reads a file with none of them as plain CSV.
Closes #907
This commit is contained in:
parent
9c66bafa52
commit
bb93d57055
3 changed files with 45 additions and 1 deletions
|
|
@ -285,6 +285,20 @@ def _extra_key_strategy(
|
|||
yield row_out
|
||||
|
||||
|
||||
# Delimiters rows_from_file() looks for when it detects the format. Without a list
|
||||
# csv.Sniffer takes any character that repeats on the sampled lines, so a single-column
|
||||
# file was split on a letter, or raised "Could not determine delimiter".
|
||||
_SNIFF_DELIMITERS = ",;\t|"
|
||||
|
||||
|
||||
def _sniff_csv_dialect(sample: str) -> type[csv.Dialect]:
|
||||
try:
|
||||
return csv.Sniffer().sniff(sample, delimiters=_SNIFF_DELIMITERS)
|
||||
except csv.Error:
|
||||
# None of them: a single-column file, read as plain CSV
|
||||
return csv.excel
|
||||
|
||||
|
||||
def rows_from_file(
|
||||
fp: BinaryIO,
|
||||
format: Format | None = None,
|
||||
|
|
@ -389,7 +403,7 @@ def rows_from_file(
|
|||
with buffered:
|
||||
return rows_from_file(buffered, format=Format.JSON)
|
||||
else:
|
||||
dialect = csv.Sniffer().sniff(
|
||||
dialect = _sniff_csv_dialect(
|
||||
first_bytes.decode(encoding or "utf-8-sig", "ignore")
|
||||
)
|
||||
rows, _ = rows_from_file(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue