sqlite-utils insert-files command, closes #122

docs/cli.rst

@@ -384,6 +384,80 @@ The command will fail if you reference columns that do not exist on the table. T
 .. note::
     ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 <https://github.com/simonw/sqlite-utils/issues/66>`__ for details of this change.
 
+.. _cli_insert_files:
+
+Inserting binary data from files
+================================
+
+SQLite ``BLOB`` columns can be used to store binary content. It can be useful to insert the contents of files into a SQLite table.
+
+The ``insert-files`` command can be used to insert the content of files, along with their metadata.
+
+Here's an example that inserts all of the GIF files in the current directory into a ``gifs.db`` database, placing the file contents in an ``images`` table::
+
+    $ sqlite-utils insert-files gifs.db images *.gif
+
+You can also pass one or more directories, in which case every file in those directories will be added recursively::
+
+    $ sqlite-utils insert-files gifs.db images path/to/my-gifs
+
+By default this command will create a table with the following schema::
+
+    CREATE TABLE [images] (
+        [path] TEXT PRIMARY KEY,
+        [content] BLOB,
+        [size] INTEGER
+    );
+
+You can customize the schema using one or more ``-c`` options. For a table schema that includes just the path, MD5 hash and last modification time of the file, you would use this::
+
+    $ sqlite-utils insert-files gifs.db images *.gif -c path -c content -c mtime --pk=path
+
+This will result in the following schema::
+
+    CREATE TABLE [images] (
+        [path] TEXT PRIMARY KEY,
+        [content] BLOB,
+        [mtime] FLOAT
+    );
+
+You can change the name of one of these columns using a ``-c colname:coldef`` parameter. To rename the ``mtime`` column to ``last_modified`` you would use this::
+
+    $ sqlite-utils insert-files gifs.db images *.gif \
+        -c path -c content -c last_modified:mtime --pk=path
+
+You can pass ``--replace`` or ``--upsert`` to indicate what should happen if you try to insert a file with an existing primary key. Pass ``--alter`` to cause any missing columns to be added to the table.
+
+The full list of column definitions you can use is as follows:
+
+``name``
+    The name of the file, e.g. ``cleo.jpg``
+``path``
+    The path to the file relative to the root folder, e.g. ``pictures/cleo.jpg``
+``fullpath``
+    The fully resolved path to the image, e.g. ``/home/simonw/pictures/cleo.jpg``
+``sha256``
+    The SHA256 hash of the file contents
+``md5``
+    The MD5 hash of the file contents
+``mode``
+    The permission bits of the file, as an integer - you may want to convert this to octal
+``content``
+    The binary file contents, which will be stored as a BLOB
+``mtime``
+    The modification time of the file, as floating point seconds since the Unix epoch
+``ctime``
+    The creation time of the file, as floating point seconds since the Unix epoch
+``mtime_int``
+    The modification time as an integer rather than a float
+``ctime_int``
+    The creation time as an integer rather than a float
+``mtime_iso``
+    The modification time as an ISO timestamp, e.g. ``2020-07-27T04:24:06.654246``
+``ctime_iso``
+    The creation time is an ISO timestamp
+``size``
+    The integer size of the file in bytes
 
 .. _cli_create_table:
 

sqlite_utils/cli.py

@@ -1,6 +1,9 @@
 import base64
 import click
 from click_default_group import DefaultGroup
+from datetime import datetime
+import hashlib
+import pathlib
 import sqlite_utils
 from sqlite_utils.db import AlterError
 import itertools
@@ -741,6 +744,109 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols)
     )
 
 
+@cli.command(name="insert-files")
+@click.argument(
+    "path",
+    type=click.Path(file_okay=True, dir_okay=False, allow_dash=False),
+    required=True,
+)
+@click.argument("table")
+@click.argument(
+    "file_or_dir",
+    nargs=-1,
+    required=True,
+    type=click.Path(file_okay=True, dir_okay=True, allow_dash=True),
+)
+@click.option(
+    "-c", "--column", type=str, multiple=True, help="Column definitions for the table",
+)
+@click.option("--pk", type=str, help="Column to use as primary key")
+@click.option("--alter", is_flag=True, help="Alter table to add missing columns")
+@click.option("--replace", is_flag=True, help="Replace files with matching primary key")
+@click.option("--upsert", is_flag=True, help="Upsert files with matching primary key")
+def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert):
+    """
+    Insert one or more files using BLOB columns in the specified table
+
+    Example usage:
+
+    \b
+    sqlite-utils insert-files pics.db images *.gif \\
+        -c name:name \\
+        -c content:content \\
+        -c content_hash:sha256 \\
+        -c created:ctime_iso \\
+        -c modified:mtime_iso \\
+        -c size:size \\
+        --pk name
+    """
+    if not column:
+        column = ["path:path", "content:content", "size:size"]
+        if not pk:
+            pk = "path"
+
+    def yield_paths_and_relative_paths():
+        for f_or_d in file_or_dir:
+            path = pathlib.Path(f_or_d)
+            if path.is_dir():
+                for subpath in path.rglob("*"):
+                    if subpath.is_file():
+                        yield subpath, subpath.relative_to(path)
+            elif path.is_file():
+                yield path, path
+
+    # Load all paths so we can show a progress bar
+    paths_and_relative_paths = list(yield_paths_and_relative_paths())
+
+    with click.progressbar(paths_and_relative_paths) as bar:
+
+        def to_insert():
+            for path, relative_path in bar:
+                row = {}
+                for coldef in column:
+                    if ":" in coldef:
+                        colname, coltype = coldef.rsplit(":", 1)
+                    else:
+                        colname, coltype = coldef, coldef
+                    try:
+                        if coltype == "path":
+                            value = str(relative_path)
+                        else:
+                            value = FILE_COLUMNS[coltype](path)
+                        row[colname] = value
+                    except KeyError:
+                        raise click.ClickException(
+                            "'{}' is not a valid column definition - options are {}".format(
+                                coltype, ", ".join(FILE_COLUMNS.keys())
+                            )
+                        )
+                yield row
+
+        db = sqlite_utils.Database(path)
+        with db.conn:
+            db[table].insert_all(
+                to_insert(), pk=pk, alter=alter, replace=replace, upsert=upsert
+            )
+
+
+FILE_COLUMNS = {
+    "name": lambda p: p.name,
+    "path": lambda p: str(p),
+    "fullpath": lambda p: str(p.resolve()),
+    "sha256": lambda p: hashlib.sha256(p.resolve().read_bytes()).hexdigest(),
+    "md5": lambda p: hashlib.md5(p.resolve().read_bytes()).hexdigest(),
+    "mode": lambda p: p.stat().st_mode,
+    "content": lambda p: p.resolve().read_bytes(),
+    "mtime": lambda p: p.stat().st_mtime,
+    "ctime": lambda p: p.stat().st_ctime,
+    "mtime_int": lambda p: int(p.stat().st_mtime),
+    "ctime_int": lambda p: int(p.stat().st_ctime),
+    "mtime_iso": lambda p: datetime.utcfromtimestamp(p.stat().st_mtime).isoformat(),
+    "ctime_iso": lambda p: datetime.utcfromtimestamp(p.stat().st_ctime).isoformat(),
+    "size": lambda p: p.stat().st_size,
+}
+
+
 def output_rows(iterator, headers, nl, arrays, json_cols):
     # We have to iterate two-at-a-time so we can know if we
     # should output a trailing comma or if we have reached

tests/test_docs.py

@@ -18,5 +18,5 @@ def documented_commands():
 
 
 @pytest.mark.parametrize("command", cli.cli.commands.keys())
-def test_plugin_hooks_are_documented(documented_commands, command):
+def test_commands_are_documented(documented_commands, command):
     assert command in documented_commands

tests/test_insert_files.py

@@ -0,0 +1,85 @@
+from sqlite_utils import cli, Database
+from click.testing import CliRunner
+import pathlib
+
+
+def test_insert_files():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        tmpdir = pathlib.Path(".")
+        print("tmpdir = ", tmpdir.resolve())
+        db_path = str(tmpdir / "files.db")
+        (tmpdir / "one.txt").write_text("This is file one", "utf-8")
+        (tmpdir / "two.txt").write_text("Two is shorter", "utf-8")
+        (tmpdir / "nested").mkdir()
+        (tmpdir / "nested" / "three.txt").write_text("Three is nested", "utf-8")
+        coltypes = (
+            "name",
+            "path",
+            "fullpath",
+            "sha256",
+            "md5",
+            "mode",
+            "content",
+            "mtime",
+            "ctime",
+            "mtime_int",
+            "ctime_int",
+            "mtime_iso",
+            "ctime_iso",
+            "size",
+        )
+        cols = []
+        for coltype in coltypes:
+            cols += ["-c", "{}:{}".format(coltype, coltype)]
+        result = runner.invoke(
+            cli.cli,
+            ["insert-files", db_path, "files", str(tmpdir)] + cols + ["--pk", "path"],
+            catch_exceptions=False,
+        )
+        assert result.exit_code == 0, result.stdout
+        db = Database(db_path)
+        rows_by_path = {r["path"]: r for r in db["files"].rows}
+        one, two, three = (
+            rows_by_path["one.txt"],
+            rows_by_path["two.txt"],
+            rows_by_path["nested/three.txt"],
+        )
+        assert {
+            "content": b"This is file one",
+            "md5": "556dfb57fce9ca301f914e2273adf354",
+            "name": "one.txt",
+            "path": "one.txt",
+            "sha256": "e34138f26b5f7368f298b4e736fea0aad87ddec69fbd04dc183b20f4d844bad5",
+            "size": 16,
+        }.items() <= one.items()
+        assert {
+            "content": b"Two is shorter",
+            "md5": "f86f067b083af1911043eb215e74ac70",
+            "name": "two.txt",
+            "path": "two.txt",
+            "sha256": "9368988ed16d4a2da0af9db9b686d385b942cb3ffd4e013f43aed2ec041183d9",
+            "size": 14,
+        }.items() <= two.items()
+        assert {
+            "content": b"Three is nested",
+            "md5": "12580f341781f5a5b589164d3cd39523",
+            "name": "three.txt",
+            "path": "nested/three.txt",
+            "sha256": "6dd45aaaaa6b9f96af19363a92c8fca5d34791d3c35c44eb19468a6a862cc8cd",
+            "size": 15,
+        }.items() <= three.items()
+        # Assert the other int/str/float columns exist and are of the right types
+        expected_types = {
+            "ctime": float,
+            "ctime_int": int,
+            "ctime_iso": str,
+            "mtime": float,
+            "mtime_int": int,
+            "mtime_iso": str,
+            "mode": int,
+            "fullpath": str,
+        }
+        for colname, expected_type in expected_types.items():
+            for row in (one, two, three):
+                assert isinstance(row[colname], expected_type)