Fix progress bar reporting incorrect percentage for multi-byte encodings

UpdateWrapper tracked progress using len(line) which counts characters,
but the progress bar total is set to the file size in bytes. For multi-byte
encodings like UTF-16-LE (2 bytes per character) the bar would only reach
50% before the file finished. Now encode each line/chunk back to bytes using
the stream's encoding to report the correct byte count.
This commit is contained in:
ikatyal2110 2026-07-15 14:18:48 +00:00
commit 9a605e058c
No known key found for this signature in database
2 changed files with 45 additions and 4 deletions

View file

@ -100,3 +100,36 @@ def test_flatten(input, expected):
)
def test_dedupe_keys(input, expected):
assert utils.dedupe_keys(input) == expected
def test_update_wrapper_byte_progress_for_multibyte_encoding():
"""UpdateWrapper should track bytes consumed, not characters (issue #439)."""
content = "hello\nworld\n"
# Build a UTF-16-LE encoded binary stream
encoded = content.encode("utf-16-le")
binary_stream = io.BytesIO(encoded)
text_stream = io.TextIOWrapper(binary_stream, encoding="utf-16-le")
updates = []
wrapper = utils.UpdateWrapper(text_stream, updates.append)
lines = list(wrapper)
assert lines == ["hello\n", "world\n"]
# Each character is 2 bytes in UTF-16-LE, so total bytes == 2 * len(content)
assert sum(updates) == len(encoded)
def test_update_wrapper_byte_progress_for_utf8():
"""UpdateWrapper byte tracking should also be correct for UTF-8."""
content = "hello\nworld\n"
encoded = content.encode("utf-8")
binary_stream = io.BytesIO(encoded)
text_stream = io.TextIOWrapper(binary_stream, encoding="utf-8")
updates = []
wrapper = utils.UpdateWrapper(text_stream, updates.append)
lines = list(wrapper)
assert lines == ["hello\n", "world\n"]
assert sum(updates) == len(encoded)