Fix all remaining resource warnings, refs #693

https://gistpreview.github.io/?0bb8e869b82f6ff0db647de755182502

sqlite_utils/cli.py

@@ -905,7 +905,7 @@ def insert_upsert_options(*, require_pk=False):
                     required=True,
                 ),
                 click.argument("table"),
-                click.argument("file", type=click.File("rb"), required=True),
+                click.argument("file", type=click.File("rb", lazy=True), required=True),
                 click.option(
                     "--pk",
                     help="Columns to use as the primary key, e.g. id",
@@ -2000,6 +2000,7 @@ def memory(
     for i, path in enumerate(paths):
         # Path may have a :format suffix
         fp = None
+        should_close_fp = False
         if ":" in path and path.rsplit(":", 1)[-1].upper() in Format.__members__:
             path, suffix = path.rsplit(":", 1)
             format = Format[suffix.upper()]
@@ -2017,29 +2018,32 @@ def memory(
                 file_table = stem
             stem_counts[stem] = stem_counts.get(stem, 1) + 1
             fp = file_path.open("rb")
-        rows, format_used = rows_from_file(fp, format=format, encoding=encoding)
-        tracker = None
-        if format_used in (Format.CSV, Format.TSV) and not no_detect_types:
-            tracker = TypeTracker()
-            rows = tracker.wrap(rows)
-        if flatten:
-            rows = (_flatten(row) for row in rows)
+            should_close_fp = True
+        try:
+            rows, format_used = rows_from_file(fp, format=format, encoding=encoding)
+            tracker = None
+            if format_used in (Format.CSV, Format.TSV) and not no_detect_types:
+                tracker = TypeTracker()
+                rows = tracker.wrap(rows)
+            if flatten:
+                rows = (_flatten(row) for row in rows)
 
-        db[file_table].insert_all(rows, alter=True)
-        if tracker is not None:
-            db[file_table].transform(types=tracker.types)
-        # Add convenient t / t1 / t2 views
-        view_names = ["t{}".format(i + 1)]
-        if i == 0:
-            view_names.append("t")
-        for view_name in view_names:
-            if not db[view_name].exists():
-                db.create_view(
-                    view_name, "select * from {}".format(quote_identifier(file_table))
-                )
-
-        if fp:
-            fp.close()
+            db[file_table].insert_all(rows, alter=True)
+            if tracker is not None:
+                db[file_table].transform(types=tracker.types)
+            # Add convenient t / t1 / t2 views
+            view_names = ["t{}".format(i + 1)]
+            if i == 0:
+                view_names.append("t")
+            for view_name in view_names:
+                if not db[view_name].exists():
+                    db.create_view(
+                        view_name,
+                        "select * from {}".format(quote_identifier(file_table)),
+                    )
+        finally:
+            if should_close_fp and fp:
+                fp.close()
 
     if analyze:
         _analyze(db, tables=None, columns=None, save=False)

sqlite_utils/utils.py

@@ -9,7 +9,29 @@ import json
 import os
 import sys
 from . import recipes
-from typing import Dict, cast, BinaryIO, Iterable, Optional, Tuple, Type
+from typing import Dict, cast, BinaryIO, Iterable, Iterator, Optional, Tuple, Type
+
+
+class _CloseableIterator(Iterator[dict]):
+    """Iterator wrapper that closes a file when iteration is complete."""
+
+    def __init__(self, iterator: Iterator[dict], closeable: io.IOBase):
+        self._iterator = iterator
+        self._closeable = closeable
+
+    def __iter__(self) -> "_CloseableIterator":
+        return self
+
+    def __next__(self) -> dict:
+        try:
+            return next(self._iterator)
+        except StopIteration:
+            self._closeable.close()
+            raise
+
+    def close(self) -> None:
+        self._closeable.close()
+
 
 import click
 
@@ -299,7 +321,8 @@ def rows_from_file(
             reader = csv.DictReader(decoded_fp, dialect=dialect)
         else:
             reader = csv.DictReader(decoded_fp)
-        return _extra_key_strategy(reader, ignore_extras, extras_key), Format.CSV
+        rows = _extra_key_strategy(reader, ignore_extras, extras_key)
+        return _CloseableIterator(iter(rows), decoded_fp), Format.CSV
     elif format == Format.TSV:
         rows = rows_from_file(
             fp, format=Format.CSV, dialect=csv.excel_tab, encoding=encoding