diff --git a/docs/cli.rst b/docs/cli.rst index 66c93cc..650da54 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1748,6 +1748,16 @@ Pass ``--sqlar`` to store the content zlib-compressed instead, using the same `` Content is only stored compressed if doing so makes it smaller - otherwise the original bytes are stored as-is, matching the behaviour of SQLite's ``sqlar_compress()`` function. +You can use ``--convert`` to transform each row before it is inserted, so imports that need custom logic - like compressing content into an archive format - don't need a separate :ref:`sqlite-utils convert ` command run against the table afterwards. It works the same way as the ``--convert`` option on :ref:`insert and upsert `, with a ``row`` variable available to modify in place: + +.. code-block:: bash + + sqlite-utils insert-files archive.db sqlar *.gif \ + -c name:name -c mode:mode -c mtime:mtime_int -c sz:size -c data:content \ + --convert 'row["data"] = zlib.compress(row["data"])' --import zlib + +As with ``sqlite-utils convert`` you can use ``--import`` to import additional Python modules, and the same :ref:`recipe functions ` are available via ``r.``. + You can customize the schema using one or more ``-c`` options. For a table schema that includes just the path, MD5 hash and last modification time of the file, you would use this: .. code-block:: bash diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 5609d65..f71b725 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2923,6 +2923,14 @@ def extract( "--encoding", help="Character encoding for input, defaults to utf-8", ) +@click.option("--convert", help="Python code to convert each row before insertion") +@click.option( + "--import", + "imports", + type=str, + multiple=True, + help="Python modules to import", +) @click.option("-s", "--silent", is_flag=True, help="Don't show a progress bar") @load_extension_option def insert_files( @@ -2938,6 +2946,8 @@ def insert_files( text, sqlar, encoding, + convert, + imports, silent, load_extension, ): @@ -2955,9 +2965,22 @@ def insert_files( -c modified:mtime_iso \\ -c size:size \\ --pk name + + Use --convert to transform each row before it is inserted, the same way + as sqlite-utils convert: + + \b + sqlite-utils insert-files archive.db sqlar *.gif --sqlar \\ + --convert 'row["data"] = zlib.compress(row["data"])' --import zlib """ if text and sqlar: raise click.ClickException("Cannot use --text and --sqlar together") + convert_fn = None + if convert: + try: + convert_fn = _compile_code(convert, imports, variable="row") + except SyntaxError as e: + raise click.ClickException(str(e)) if not column: if text: column = ["path:path", "content_text:content_text", "size:size"] @@ -3051,6 +3074,8 @@ def insert_files( # Special case for --name if coltype == "name" and name: row[colname] = name + if convert_fn is not None: + row = convert_fn(row) or row yield row db = sqlite_utils.Database(path) diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 9a5278d..d59043a 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -207,6 +207,60 @@ def test_insert_files_sqlar(): assert two["data"] == incompressible +def test_insert_files_convert(): + # Same effect as --sqlar, but implemented using --convert against + # plain content/size columns - no separate `convert` command needed. + runner = CliRunner() + with runner.isolated_filesystem(): + tmpdir = pathlib.Path(".") + db_path = str(tmpdir / "files.db") + compressible = b"abcdefgh" * 1000 + (tmpdir / "one.txt").write_bytes(compressible) + result = runner.invoke( + cli.cli, + [ + "insert-files", + db_path, + "files", + str(tmpdir / "one.txt"), + "-c", + "name:name", + "-c", + "data:content", + "--convert", + 'row["data"] = zlib.compress(row["data"])', + "--import", + "zlib", + ], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + row = list(db["files"].rows)[0] + assert zlib.decompress(row["data"]) == compressible + + +def test_insert_files_convert_syntax_error(): + runner = CliRunner() + with runner.isolated_filesystem(): + tmpdir = pathlib.Path(".") + db_path = str(tmpdir / "files.db") + (tmpdir / "one.txt").write_text("hello", "utf-8") + result = runner.invoke( + cli.cli, + [ + "insert-files", + db_path, + "files", + str(tmpdir), + "--convert", + "def broken(:", + ], + ) + assert result.exit_code == 1 + assert "Could not compile code" in result.output + + def test_insert_files_sqlar_and_text_conflict(): runner = CliRunner() with runner.isolated_filesystem():