insert-files can now read from stdin, closes #127

docs/cli.rst

@@ -459,6 +459,14 @@ The full list of column definitions you can use is as follows:
 ``size``
     The integer size of the file in bytes
 
+You can insert data piped from standard input like this::
+
+    cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg
+
+The ``-`` argument indicates data should be read from standard input. The string passed using the ``--name`` option will be used for the file name and path values.
+
+When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``sha256``, ``md5`` and ``size``.
+
 .. _cli_create_table:
 
 Creating tables

sqlite_utils/cli.py

@@ -764,7 +764,8 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols)
 @click.option("--alter", is_flag=True, help="Alter table to add missing columns")
 @click.option("--replace", is_flag=True, help="Replace files with matching primary key")
 @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key")
-def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert):
+@click.option("--name", type=str, help="File name to use")
+def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert, name):
     """
     Insert one or more files using BLOB columns in the specified table
 
@@ -788,7 +789,9 @@ def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert):
     def yield_paths_and_relative_paths():
         for f_or_d in file_or_dir:
             path = pathlib.Path(f_or_d)
-            if path.is_dir():
+            if f_or_d == "-":
+                yield "-", "-"
+            elif path.is_dir():
                 for subpath in path.rglob("*"):
                     if subpath.is_file():
                         yield subpath, subpath.relative_to(path)
@@ -803,23 +806,35 @@ def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert):
         def to_insert():
             for path, relative_path in bar:
                 row = {}
+                lookups = FILE_COLUMNS
+                if path == "-":
+                    stdin_data = sys.stdin.buffer.read()
+                    # We only support a subset of columns for this case
+                    lookups = {
+                        "name": lambda p: name or "-",
+                        "path": lambda p: name or "-",
+                        "content": lambda p: stdin_data,
+                        "sha256": lambda p: hashlib.sha256(stdin_data).hexdigest(),
+                        "md5": lambda p: hashlib.md5(stdin_data).hexdigest(),
+                        "size": lambda p: len(stdin_data),
+                    }
                 for coldef in column:
                     if ":" in coldef:
                         colname, coltype = coldef.rsplit(":", 1)
                     else:
                         colname, coltype = coldef, coldef
                     try:
-                        if coltype == "path":
-                            value = str(relative_path)
-                        else:
-                            value = FILE_COLUMNS[coltype](path)
+                        value = lookups[coltype](path)
                         row[colname] = value
                     except KeyError:
                         raise click.ClickException(
                             "'{}' is not a valid column definition - options are {}".format(
-                                coltype, ", ".join(FILE_COLUMNS.keys())
+                                coltype, ", ".join(lookups.keys())
                             )
                         )
+                    # Special case for --name
+                    if coltype == "name" and name:
+                        row[colname] = name
                 yield row
 
         db = sqlite_utils.Database(path)

tests/test_insert_files.py

@@ -7,7 +7,6 @@ def test_insert_files():
     runner = CliRunner()
     with runner.isolated_filesystem():
         tmpdir = pathlib.Path(".")
-        print("tmpdir = ", tmpdir.resolve())
         db_path = str(tmpdir / "files.db")
         (tmpdir / "one.txt").write_text("This is file one", "utf-8")
         (tmpdir / "two.txt").write_text("Two is shorter", "utf-8")
@@ -83,3 +82,20 @@ def test_insert_files():
         for colname, expected_type in expected_types.items():
             for row in (one, two, three):
                 assert isinstance(row[colname], expected_type)
+
+
+def test_insert_files_stdin():
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        tmpdir = pathlib.Path(".")
+        db_path = str(tmpdir / "files.db")
+        result = runner.invoke(
+            cli.cli,
+            ["insert-files", db_path, "files", "-", "--name", "stdin-name"],
+            catch_exceptions=False,
+            input="hello world",
+        )
+        assert result.exit_code == 0, result.stdout
+        db = Database(db_path)
+        row = list(db["files"].rows)[0]
+        assert {"path": "stdin-name", "content": b"hello world", "size": 11} == row