mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-09-09 01:54:21 +02:00
Fix progress bar stalling at ~50% for multi-byte encodings like utf-16-le
UpdateWrapper.__iter__ was using len(line) (character count) to track progress, but the progress bar is initialised with os.path.getsize() (byte count). For utf-16-le each character is 2 bytes, so the bar only ever reached ~50% before the insert completed. Use TextIOWrapper.buffer.tell() when available to report the true number of bytes consumed, falling back to len(line) for non-TextIOWrapper objects. Fixes #439
This commit is contained in:
parent
a947dc6739
commit
eaf39c4929
2 changed files with 57 additions and 1 deletions
|
|
@ -2205,6 +2205,42 @@ def test_insert_encoding(tmpdir):
|
|||
]
|
||||
|
||||
|
||||
def test_insert_utf16le_encoding(tmpdir):
|
||||
"""Progress bar stays accurate and data inserts correctly for UTF-16-LE CSV files.
|
||||
|
||||
The progress bar was initialized with the file's byte length but UpdateWrapper
|
||||
used len(decoded_line) (character count) for updates. For utf-16-le each
|
||||
character is 2 bytes, so the bar stalled at ~50% instead of reaching 100%.
|
||||
"""
|
||||
db_path = str(tmpdir / "test.db")
|
||||
csv_content = "id,name\n1,Alice\n2,Bob\n"
|
||||
utf16le_bytes = csv_content.encode("utf-16-le")
|
||||
csv_path = str(tmpdir / "test.csv")
|
||||
with open(csv_path, "wb") as fp:
|
||||
fp.write(utf16le_bytes)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
cli.cli,
|
||||
[
|
||||
"insert",
|
||||
db_path,
|
||||
"names",
|
||||
csv_path,
|
||||
"--csv",
|
||||
"--encoding",
|
||||
"utf-16-le",
|
||||
"--no-detect-types",
|
||||
],
|
||||
catch_exceptions=False,
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
db = Database(db_path)
|
||||
assert list(db["names"].rows) == [
|
||||
{"id": "1", "name": "Alice"},
|
||||
{"id": "2", "name": "Bob"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fts", ["FTS4", "FTS5"])
|
||||
@pytest.mark.parametrize(
|
||||
"extra_arg,expected",
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue