Fix progress bar stalling at ~50% for multi-byte encodings like utf-16-le

UpdateWrapper.__iter__ was using len(line) (character count) to track
progress, but the progress bar is initialised with os.path.getsize()
(byte count).  For utf-16-le each character is 2 bytes, so the bar
only ever reached ~50% before the insert completed.

Use TextIOWrapper.buffer.tell() when available to report the true
number of bytes consumed, falling back to len(line) for non-TextIOWrapper
objects.

Fixes #439
This commit is contained in:
Claude 2026-07-24 14:29:42 +00:00
commit eaf39c4929
No known key found for this signature in database
2 changed files with 57 additions and 1 deletions

View file

@ -212,8 +212,28 @@ class UpdateWrapper:
self._update = update
def __iter__(self) -> Iterator[bytes]:
# For TextIOWrapper objects, use the underlying binary buffer position
# to track bytes consumed rather than character count. This matters for
# multi-byte encodings (e.g. utf-16-le) where len(line) is roughly half
# the actual byte count, causing the progress bar to stall at ~50%.
binary = getattr(self._wrapped, "buffer", None)
last_pos: Optional[int] = None
if binary is not None:
try:
last_pos = binary.tell()
except OSError:
binary = None
for line in self._wrapped:
self._update(len(line))
if binary is not None:
try:
pos = binary.tell()
self._update(pos - last_pos)
last_pos = pos
except OSError:
self._update(len(line))
binary = None
else:
self._update(len(line))
yield line
def read(self, size: int = -1) -> bytes: