progressbar for inserts/upserts of other file formats

* progressbar for inserts/upserts of other file formats, closes #485
* Pin to Python 3.10.6 for the moment as workaround for mypy error

Co-authored-by: Simon Willison <swillison@gmail.com>

.github/workflows/publish.yml

@@ -9,12 +9,12 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10.6"]
         os: [ubuntu-latest, windows-latest, macos-latest]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - uses: actions/cache@v2

.github/workflows/test.yml

@@ -10,13 +10,13 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10.6"]
         numpy: [0, 1]
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
     - uses: actions/cache@v2

sqlite_utils/cli.py

@@ -953,14 +953,16 @@ def insert_upsert_implementation(
         decoded = io.TextIOWrapper(file, encoding=encoding)
 
     tracker = None
-    if csv or tsv:
+    with file_progress(decoded, silent=silent) as decoded:
-        if sniff:
+        if csv or tsv:
-            # Read first 2048 bytes and use that to detect
+            if sniff:
-            first_bytes = sniff_buffer.peek(2048)
+                # Read first 2048 bytes and use that to detect
-            dialect = csv_std.Sniffer().sniff(first_bytes.decode(encoding, "ignore"))
+                first_bytes = sniff_buffer.peek(2048)
-        else:
+                dialect = csv_std.Sniffer().sniff(
-            dialect = "excel-tab" if tsv else "excel"
+                    first_bytes.decode(encoding, "ignore")
-        with file_progress(decoded, silent=silent) as decoded:
+                )
+            else:
+                dialect = "excel-tab" if tsv else "excel"
             csv_reader_args = {"dialect": dialect}
             if delimiter:
                 csv_reader_args["delimiter"] = delimiter
@@ -977,24 +979,24 @@ def insert_upsert_implementation(
             if detect_types:
                 tracker = TypeTracker()
                 docs = tracker.wrap(docs)
-    elif lines:
+        elif lines:
-        docs = ({"line": line.strip()} for line in decoded)
+            docs = ({"line": line.strip()} for line in decoded)
-    elif text:
+        elif text:
-        docs = ({"text": decoded.read()},)
+            docs = ({"text": decoded.read()},)
-    else:
+        else:
-        try:
+            try:
-            if nl:
+                if nl:
-                docs = (json.loads(line) for line in decoded if line.strip())
+                    docs = (json.loads(line) for line in decoded if line.strip())
-            else:
+                else:
-                docs = json.load(decoded)
+                    docs = json.load(decoded)
-                if isinstance(docs, dict):
+                    if isinstance(docs, dict):
-                    docs = [docs]
+                        docs = [docs]
-        except json.decoder.JSONDecodeError:
+            except json.decoder.JSONDecodeError:
-            raise click.ClickException(
+                raise click.ClickException(
-                "Invalid JSON - use --csv for CSV or --tsv for TSV files"
+                    "Invalid JSON - use --csv for CSV or --tsv for TSV files"
-            )
+                )
-        if flatten:
+            if flatten:
-            docs = (dict(_flatten(doc)) for doc in docs)
+                docs = (dict(_flatten(doc)) for doc in docs)
 
     if convert:
         variable = "row"

sqlite_utils/utils.py

@@ -155,6 +155,11 @@ class UpdateWrapper:
             self._update(len(line))
             yield line
 
+    def read(self, size=-1):
+        data = self._wrapped.read(size)
+        self._update(len(data))
+        return data
+
 
 @contextlib.contextmanager
 def file_progress(file, silent=False, **kwargs):