mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-09-07 17:14:09 +02:00
Fix progress bar stalling at ~50% for multi-byte encodings like utf-16-le
UpdateWrapper.__iter__ was using len(line) (character count) to track progress, but the progress bar is initialised with os.path.getsize() (byte count). For utf-16-le each character is 2 bytes, so the bar only ever reached ~50% before the insert completed. Use TextIOWrapper.buffer.tell() when available to report the true number of bytes consumed, falling back to len(line) for non-TextIOWrapper objects. Fixes #439
This commit is contained in:
parent
a947dc6739
commit
eaf39c4929
2 changed files with 57 additions and 1 deletions
|
|
@ -212,8 +212,28 @@ class UpdateWrapper:
|
|||
self._update = update
|
||||
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
# For TextIOWrapper objects, use the underlying binary buffer position
|
||||
# to track bytes consumed rather than character count. This matters for
|
||||
# multi-byte encodings (e.g. utf-16-le) where len(line) is roughly half
|
||||
# the actual byte count, causing the progress bar to stall at ~50%.
|
||||
binary = getattr(self._wrapped, "buffer", None)
|
||||
last_pos: Optional[int] = None
|
||||
if binary is not None:
|
||||
try:
|
||||
last_pos = binary.tell()
|
||||
except OSError:
|
||||
binary = None
|
||||
for line in self._wrapped:
|
||||
self._update(len(line))
|
||||
if binary is not None:
|
||||
try:
|
||||
pos = binary.tell()
|
||||
self._update(pos - last_pos)
|
||||
last_pos = pos
|
||||
except OSError:
|
||||
self._update(len(line))
|
||||
binary = None
|
||||
else:
|
||||
self._update(len(line))
|
||||
yield line
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue