Fix progress bar reporting character count instead of byte count for multibyte encodings

UpdateWrapper.__iter__ and .read() used len(line) which counts decoded characters,
but the progress bar total comes from os.path.getsize() (raw bytes). For encodings
like utf-16-le each character is 2 bytes, so the bar would only reach ~50%. Encode
the string back with the file's encoding to get the correct byte length.

Fixes #439
This commit is contained in:
Claude 2026-07-22 14:21:31 +00:00
commit c6a84225a2
No known key found for this signature in database
2 changed files with 37 additions and 2 deletions

View file

@ -212,13 +212,15 @@ class UpdateWrapper:
self._update = update
def __iter__(self) -> Iterator[bytes]:
encoding = getattr(self._wrapped, "encoding", None) or "utf-8"
for line in self._wrapped:
self._update(len(line))
self._update(len(line.encode(encoding, errors="replace")))
yield line
def read(self, size: int = -1) -> bytes:
data = self._wrapped.read(size)
self._update(len(data))
encoding = getattr(self._wrapped, "encoding", None) or "utf-8"
self._update(len(data.encode(encoding, errors="replace")))
return data