remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-09-17 05:54:23 +02:00
Code Issues Projects Releases Packages Wiki Activity

progressbar for inserts/upserts of all fileformats, closes #485

Browse source
This commit is contained in:
Mischa Untaga 2022-09-08 16:56:52 +02:00
commit 5fd78e27d2
2 changed files with 31 additions and 26 deletions
Download patch file Download diff file Expand all files Collapse all files

52
sqlite_utils/cli.py
View file

@ -953,14 +953,14 @@ def insert_upsert_implementation(
decoded = io.TextIOWrapper(file, encoding=encoding) decoded = io.TextIOWrapper(file, encoding=encoding)
tracker = None tracker = None
if csv or tsv: with file_progress(decoded, silent=silent) as decoded:
if sniff: if csv or tsv:
# Read first 2048 bytes and use that to detect if sniff:
first_bytes = sniff_buffer.peek(2048) # Read first 2048 bytes and use that to detect
dialect = csv_std.Sniffer().sniff(first_bytes.decode(encoding, "ignore")) first_bytes = sniff_buffer.peek(2048)
else: dialect = csv_std.Sniffer().sniff(first_bytes.decode(encoding, "ignore"))
dialect = "excel-tab" if tsv else "excel" else:
with file_progress(decoded, silent=silent) as decoded: dialect = "excel-tab" if tsv else "excel"
csv_reader_args = {"dialect": dialect} csv_reader_args = {"dialect": dialect}
if delimiter: if delimiter:
csv_reader_args["delimiter"] = delimiter csv_reader_args["delimiter"] = delimiter
@ -977,24 +977,24 @@ def insert_upsert_implementation(
if detect_types: if detect_types:
tracker = TypeTracker() tracker = TypeTracker()
docs = tracker.wrap(docs) docs = tracker.wrap(docs)
elif lines: elif lines:
docs = ({"line": line.strip()} for line in decoded) docs = ({"line": line.strip()} for line in decoded)
elif text: elif text:
docs = ({"text": decoded.read()},) docs = ({"text": decoded.read()},)
else: else:
try: try:
if nl: if nl:
docs = (json.loads(line) for line in decoded if line.strip()) docs = (json.loads(line) for line in decoded if line.strip())
else: else:
docs = json.load(decoded) docs = json.load(decoded)
if isinstance(docs, dict): if isinstance(docs, dict):
docs = [docs] docs = [docs]
except json.decoder.JSONDecodeError: except json.decoder.JSONDecodeError:
raise click.ClickException( raise click.ClickException(
"Invalid JSON - use --csv for CSV or --tsv for TSV files" "Invalid JSON - use --csv for CSV or --tsv for TSV files"
) )
if flatten: if flatten:
docs = (dict(_flatten(doc)) for doc in docs) docs = (dict(_flatten(doc)) for doc in docs)
if convert: if convert:
variable = "row" variable = "row"

5
sqlite_utils/utils.py
View file

@ -155,6 +155,11 @@ class UpdateWrapper:
self._update(len(line)) self._update(len(line))
yield line yield line
def read(self, size=-1):
data = self._wrapped.read(size)
self._update(len(data))
return data
@contextlib.contextmanager @contextlib.contextmanager
def file_progress(file, silent=False, **kwargs): def file_progress(file, silent=False, **kwargs):