insert-files: add --convert/--import to transform rows during import

Reuses the same _compile_code mechanism as insert/upsert --convert and the
convert command (row variable, r. recipes, --import for extra modules)
instead of a bespoke mini-language. Lets sqlar-style imports (or any other
per-row transform) happen in one insert-files call rather than needing a
separate convert pass afterwards.

Closes #597

docs/cli.rst

@@ -1748,6 +1748,16 @@ Pass ``--sqlar`` to store the content zlib-compressed instead, using the same ``
 
 Content is only stored compressed if doing so makes it smaller - otherwise the original bytes are stored as-is, matching the behaviour of SQLite's ``sqlar_compress()`` function.
 
+You can use ``--convert`` to transform each row before it is inserted, so imports that need custom logic - like compressing content into an archive format - don't need a separate :ref:`sqlite-utils convert <cli_convert>` command run against the table afterwards. It works the same way as the ``--convert`` option on :ref:`insert and upsert <cli_insert_convert>`, with a ``row`` variable available to modify in place:
+
+.. code-block:: bash
+
+    sqlite-utils insert-files archive.db sqlar *.gif \
+        -c name:name -c mode:mode -c mtime:mtime_int -c sz:size -c data:content \
+        --convert 'row["data"] = zlib.compress(row["data"])' --import zlib
+
+As with ``sqlite-utils convert`` you can use ``--import`` to import additional Python modules, and the same :ref:`recipe functions <cli_convert_recipes>` are available via ``r.``.
+
 You can customize the schema using one or more ``-c`` options. For a table schema that includes just the path, MD5 hash and last modification time of the file, you would use this:
 
 .. code-block:: bash

sqlite_utils/cli.py

@@ -2923,6 +2923,14 @@ def extract(
     "--encoding",
     help="Character encoding for input, defaults to utf-8",
 )
+@click.option("--convert", help="Python code to convert each row before insertion")
+@click.option(
+    "--import",
+    "imports",
+    type=str,
+    multiple=True,
+    help="Python modules to import",
+)
 @click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar")
 @load_extension_option
 def insert_files(
@@ -2938,6 +2946,8 @@ def insert_files(
     text,
     sqlar,
     encoding,
+    convert,
+    imports,
     silent,
     load_extension,
 ):
@@ -2955,9 +2965,22 @@ def insert_files(
             -c modified:mtime_iso \\
             -c size:size \\
             --pk name
+
+    Use --convert to transform each row before it is inserted, the same way
+    as sqlite-utils convert:
+
+    \b
+        sqlite-utils insert-files archive.db sqlar *.gif --sqlar \\
+            --convert 'row["data"] = zlib.compress(row["data"])' --import zlib
     """
     if text and sqlar:
         raise click.ClickException("Cannot use --text and --sqlar together")
+    convert_fn = None
+    if convert:
+        try:
+            convert_fn = _compile_code(convert, imports, variable="row")
+        except SyntaxError as e:
+            raise click.ClickException(str(e))
     if not column:
         if text:
             column = ["path:path", "content_text:content_text", "size:size"]
@@ -3051,6 +3074,8 @@ def insert_files(
                     # Special case for --name
                     if coltype == "name" and name:
                         row[colname] = name
+                if convert_fn is not None:
+                    row = convert_fn(row) or row
                 yield row
 
         db = sqlite_utils.Database(path)

tests/test_insert_files.py

@@ -207,6 +207,60 @@ def test_insert_files_sqlar():
         assert two["data"] == incompressible
 
 
+def test_insert_files_convert():
+    # Same effect as --sqlar, but implemented using --convert against
+    # plain content/size columns - no separate `convert` command needed.
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        tmpdir = pathlib.Path(".")
+        db_path = str(tmpdir / "files.db")
+        compressible = b"abcdefgh" * 1000
+        (tmpdir / "one.txt").write_bytes(compressible)
+        result = runner.invoke(
+            cli.cli,
+            [
+                "insert-files",
+                db_path,
+                "files",
+                str(tmpdir / "one.txt"),
+                "-c",
+                "name:name",
+                "-c",
+                "data:content",
+                "--convert",
+                'row["data"] = zlib.compress(row["data"])',
+                "--import",
+                "zlib",
+            ],
+            catch_exceptions=False,
+        )
+        assert result.exit_code == 0, result.output
+        db = Database(db_path)
+        row = list(db["files"].rows)[0]
+        assert zlib.decompress(row["data"]) == compressible
+
+
+def test_insert_files_convert_syntax_error():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        tmpdir = pathlib.Path(".")
+        db_path = str(tmpdir / "files.db")
+        (tmpdir / "one.txt").write_text("hello", "utf-8")
+        result = runner.invoke(
+            cli.cli,
+            [
+                "insert-files",
+                db_path,
+                "files",
+                str(tmpdir),
+                "--convert",
+                "def broken(:",
+            ],
+        )
+        assert result.exit_code == 1
+        assert "Could not compile code" in result.output
+
+
 def test_insert_files_sqlar_and_text_conflict():
     runner = CliRunner()
     with runner.isolated_filesystem():