mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-09-08 09:34:08 +02:00
Fix dishonest progress accounting for multi-byte encoded CSV input (#439)
UpdateWrapper counted decoded characters (len(line)) against a byte-based file length, so utf-16-le input (2 bytes/char) only ever reported ~50% progress even after reading finished, looking like a stall. Track the underlying binary stream's position instead so progress is honest for any encoding, and add regression tests covering a clean utf-16-le insert plus a genuinely corrupt one that must still raise a visible error.
This commit is contained in:
parent
050f11f31f
commit
c989cdf5a2
3 changed files with 108 additions and 2 deletions
|
|
@ -202,15 +202,32 @@ class UpdateWrapper:
|
|||
def __init__(self, wrapped: io.IOBase, update: Callable[[int], None]) -> None:
|
||||
self._wrapped = wrapped
|
||||
self._update = update
|
||||
# If wrapped is a decoded text stream (e.g. io.TextIOWrapper set up
|
||||
# with --encoding), len(line)/len(data) counts characters, not the
|
||||
# bytes actually consumed from the underlying file - for a multi-byte
|
||||
# encoding like UTF-16 that undercounts by ~2x, so progress never
|
||||
# reaches 100% even though reading completed. Track the underlying
|
||||
# binary buffer's position instead so progress stays honest for any
|
||||
# encoding.
|
||||
self._byte_source = getattr(wrapped, "buffer", None)
|
||||
self._last_pos = self._byte_source.tell() if self._byte_source else 0
|
||||
|
||||
def _report(self, data_len: int) -> None:
|
||||
if self._byte_source is not None:
|
||||
pos = self._byte_source.tell()
|
||||
self._update(pos - self._last_pos)
|
||||
self._last_pos = pos
|
||||
else:
|
||||
self._update(data_len)
|
||||
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
for line in self._wrapped:
|
||||
self._update(len(line))
|
||||
self._report(len(line))
|
||||
yield line
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
data = self._wrapped.read(size)
|
||||
self._update(len(data))
|
||||
self._report(len(data))
|
||||
return data
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue