Forward ignore_extras= and extras_key= from the TSV and auto-detect paths

Both branches that recurse into the CSV branch called rows_from_file()
without forwarding the two arguments, so the inner call applied its own
defaults and raised RowError even when the caller had explicitly asked
to ignore or capture the extras. The outer _extra_key_strategy wrapper
in each branch was dead code as a result.

Forward both arguments and drop the redundant wrapper, so the strategy
is applied once on the shared CSV path. With no options passed the
behaviour is unchanged.

Fixes #892
This commit is contained in:
feiiiiii5 2026-09-30 19:29:23 +08:00
commit d5d01b3f9e
2 changed files with 50 additions and 18 deletions

View file

@ -358,17 +358,18 @@ def rows_from_file(
)
return _CloseableIterator(iter(rows), decoded_fp), Format.CSV
elif format == Format.TSV:
# The inner call applies the extra-field strategy, so the
# ignore_extras= and extras_key= arguments must be passed to it -
# see https://github.com/simonw/sqlite-utils/issues/892
rows, _ = rows_from_file(
fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding
)
return (
_extra_key_strategy(
cast(Iterable[dict[str | None, object]], rows),
ignore_extras,
extras_key,
),
Format.TSV,
fp,
format=Format.CSV,
dialect=csv.excel_tab,
encoding=encoding,
ignore_extras=ignore_extras,
extras_key=extras_key,
)
return rows, Format.TSV
elif format is None:
# Detect the format, then call this recursively
buffered = io.BufferedReader(cast(io.RawIOBase, fp), buffer_size=4096)
@ -393,18 +394,16 @@ def rows_from_file(
first_bytes.decode(encoding or "utf-8-sig", "ignore")
)
rows, _ = rows_from_file(
buffered, format=Format.CSV, dialect=dialect, encoding=encoding
buffered,
format=Format.CSV,
dialect=dialect,
encoding=encoding,
ignore_extras=ignore_extras,
extras_key=extras_key,
)
# Make sure we return the format we detected
detected_format = Format.TSV if dialect.delimiter == "\t" else Format.CSV
return (
_extra_key_strategy(
cast(Iterable[dict[str | None, object]], rows),
ignore_extras,
extras_key,
),
detected_format,
)
return rows, detected_format
else:
raise RowsFromFileError("Bad format")

View file

@ -57,6 +57,39 @@ def test_rows_from_file_extra_fields_strategies(ignore_extras, extras_key, expec
assert list_rows == expected
@pytest.mark.parametrize(
"ignore_extras,extras_key,expected",
(
(True, None, [{"id": "1", "name": "Cleo"}]),
(False, "_rest", [{"id": "1", "name": "Cleo", "_rest": ["oops"]}]),
# expected of None means expect an error:
(False, False, None),
),
)
def test_rows_from_file_tsv_extra_fields_strategies(
ignore_extras, extras_key, expected
):
# ignore_extras= and extras_key= must apply to TSV as well as CSV,
# see https://github.com/simonw/sqlite-utils/issues/892
try:
rows, detected_format = rows_from_file(
BytesIO(b"id\tname\r\n1\tCleo\toops"),
format=Format.TSV,
ignore_extras=ignore_extras,
extras_key=extras_key,
)
list_rows = list(rows)
except RowError:
if expected is None:
# This is fine,
return
else:
# We did not expect an error
raise
assert detected_format == Format.TSV
assert list_rows == expected
def test_rows_from_file_error_on_string_io():
with pytest.raises(TypeError) as ex:
rows_from_file(StringIO("id,name\r\n1,Cleo")) # type: ignore[arg-type]