mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-09-11 02:54:23 +02:00
Fix dishonest progress accounting for multi-byte encoded CSV input (#439)
UpdateWrapper counted decoded characters (len(line)) against a byte-based file length, so utf-16-le input (2 bytes/char) only ever reported ~50% progress even after reading finished, looking like a stall. Track the underlying binary stream's position instead so progress is honest for any encoding, and add regression tests covering a clean utf-16-le insert plus a genuinely corrupt one that must still raise a visible error.
This commit is contained in:
parent
050f11f31f
commit
c989cdf5a2
3 changed files with 108 additions and 2 deletions
|
|
@ -2259,6 +2259,70 @@ def test_insert_encoding(tmpdir):
|
|||
]
|
||||
|
||||
|
||||
def test_insert_encoding_utf16le(tmpdir):
|
||||
# Regression test for issue #439: against utf-16-le CSV input, progress
|
||||
# accounting used to undercount (it measured decoded characters against
|
||||
# a byte-based file length, so it topped out around 50% - looking like a
|
||||
# stall) even on a fully successful insert. And a genuine decode error
|
||||
# further into the file needs to still surface as a clear exception
|
||||
# rather than being silently swallowed.
|
||||
db_path = str(tmpdir / "test.db")
|
||||
csv_path = str(tmpdir / "test.csv")
|
||||
rows = ["id,name"] + [f"{i},Name {i}" for i in range(500)]
|
||||
with open(csv_path, "wb") as fp:
|
||||
fp.write("\n".join(rows).encode("utf-16-le"))
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli.cli,
|
||||
[
|
||||
"insert",
|
||||
db_path,
|
||||
"names",
|
||||
csv_path,
|
||||
"--csv",
|
||||
"--encoding",
|
||||
"utf-16-le",
|
||||
"--no-detect-types",
|
||||
],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
db = Database(db_path)
|
||||
# All 500 data rows made it in - progress accounting didn't drop any
|
||||
assert db["names"].count == 500
|
||||
assert list(db["names"].rows)[:2] == [
|
||||
{"id": "0", "name": "Name 0"},
|
||||
{"id": "1", "name": "Name 1"},
|
||||
]
|
||||
|
||||
# A file that is genuinely invalid utf-16-le should raise a visible,
|
||||
# descriptive error - not hang or fail silently.
|
||||
bad_csv_path = str(tmpdir / "bad.csv")
|
||||
good_bytes = "\n".join(rows).encode("utf-16-le")
|
||||
# Splice in an unpaired surrogate code unit part-way through the file,
|
||||
# aligned to a 2-byte utf-16 code unit boundary
|
||||
midpoint = (len(good_bytes) // 2) & ~1
|
||||
bad_bytes = good_bytes[:midpoint] + b"\x00\xd8" + good_bytes[midpoint:]
|
||||
with open(bad_csv_path, "wb") as fp:
|
||||
fp.write(bad_bytes)
|
||||
|
||||
bad_result = CliRunner().invoke(
|
||||
cli.cli,
|
||||
[
|
||||
"insert",
|
||||
str(tmpdir / "bad.db"),
|
||||
"names",
|
||||
bad_csv_path,
|
||||
"--csv",
|
||||
"--encoding",
|
||||
"utf-16-le",
|
||||
],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert bad_result.exit_code == 1
|
||||
assert "codec can't decode" in bad_result.output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fts", ["FTS4", "FTS5"])
|
||||
@pytest.mark.parametrize(
|
||||
"extra_arg,expected",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue