mirror of
https://github.com/simonw/sqlite-utils.git
synced 2026-08-02 07:24:21 +02:00
Fix progress bar reporting character count instead of byte count for multibyte encodings
UpdateWrapper.__iter__ and .read() used len(line) which counts decoded characters, but the progress bar total comes from os.path.getsize() (raw bytes). For encodings like utf-16-le each character is 2 bytes, so the bar would only reach ~50%. Encode the string back with the file's encoding to get the correct byte length. Fixes #439
This commit is contained in:
parent
a947dc6739
commit
c6a84225a2
2 changed files with 37 additions and 2 deletions
|
|
@ -212,13 +212,15 @@ class UpdateWrapper:
|
|||
self._update = update
|
||||
|
||||
def __iter__(self) -> Iterator[bytes]:
|
||||
encoding = getattr(self._wrapped, "encoding", None) or "utf-8"
|
||||
for line in self._wrapped:
|
||||
self._update(len(line))
|
||||
self._update(len(line.encode(encoding, errors="replace")))
|
||||
yield line
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
data = self._wrapped.read(size)
|
||||
self._update(len(data))
|
||||
encoding = getattr(self._wrapped, "encoding", None) or "utf-8"
|
||||
self._update(len(data.encode(encoding, errors="replace")))
|
||||
return data
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue