diff --git a/docs/cli.rst b/docs/cli.rst index 67714a5..a9adf63 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -459,6 +459,14 @@ The full list of column definitions you can use is as follows: ``size`` The integer size of the file in bytes +You can insert data piped from standard input like this:: + + cat dog.jpg | sqlite-utils insert-files dogs.db pics - --name=dog.jpg + +The ``-`` argument indicates data should be read from standard input. The string passed using the ``--name`` option will be used for the file name and path values. + +When inserting data from standard input only the following column definitions are supported: ``name``, ``path``, ``content``, ``sha256``, ``md5`` and ``size``. + .. _cli_create_table: Creating tables diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 3dc40ea..c60fd01 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -764,7 +764,8 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols) @click.option("--alter", is_flag=True, help="Alter table to add missing columns") @click.option("--replace", is_flag=True, help="Replace files with matching primary key") @click.option("--upsert", is_flag=True, help="Upsert files with matching primary key") -def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert): +@click.option("--name", type=str, help="File name to use") +def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert, name): """ Insert one or more files using BLOB columns in the specified table @@ -788,7 +789,9 @@ def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert): def yield_paths_and_relative_paths(): for f_or_d in file_or_dir: path = pathlib.Path(f_or_d) - if path.is_dir(): + if f_or_d == "-": + yield "-", "-" + elif path.is_dir(): for subpath in path.rglob("*"): if subpath.is_file(): yield subpath, subpath.relative_to(path) @@ -803,23 +806,35 @@ def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert): def to_insert(): for path, relative_path in bar: row = {} + lookups = FILE_COLUMNS + if path == "-": + stdin_data = sys.stdin.buffer.read() + # We only support a subset of columns for this case + lookups = { + "name": lambda p: name or "-", + "path": lambda p: name or "-", + "content": lambda p: stdin_data, + "sha256": lambda p: hashlib.sha256(stdin_data).hexdigest(), + "md5": lambda p: hashlib.md5(stdin_data).hexdigest(), + "size": lambda p: len(stdin_data), + } for coldef in column: if ":" in coldef: colname, coltype = coldef.rsplit(":", 1) else: colname, coltype = coldef, coldef try: - if coltype == "path": - value = str(relative_path) - else: - value = FILE_COLUMNS[coltype](path) + value = lookups[coltype](path) row[colname] = value except KeyError: raise click.ClickException( "'{}' is not a valid column definition - options are {}".format( - coltype, ", ".join(FILE_COLUMNS.keys()) + coltype, ", ".join(lookups.keys()) ) ) + # Special case for --name + if coltype == "name" and name: + row[colname] = name yield row db = sqlite_utils.Database(path) diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py index 7a0dd8a..f17f279 100644 --- a/tests/test_insert_files.py +++ b/tests/test_insert_files.py @@ -7,7 +7,6 @@ def test_insert_files(): runner = CliRunner() with runner.isolated_filesystem(): tmpdir = pathlib.Path(".") - print("tmpdir = ", tmpdir.resolve()) db_path = str(tmpdir / "files.db") (tmpdir / "one.txt").write_text("This is file one", "utf-8") (tmpdir / "two.txt").write_text("Two is shorter", "utf-8") @@ -83,3 +82,20 @@ def test_insert_files(): for colname, expected_type in expected_types.items(): for row in (one, two, three): assert isinstance(row[colname], expected_type) + + +def test_insert_files_stdin(): + runner = CliRunner() + with runner.isolated_filesystem(): + tmpdir = pathlib.Path(".") + db_path = str(tmpdir / "files.db") + result = runner.invoke( + cli.cli, + ["insert-files", db_path, "files", "-", "--name", "stdin-name"], + catch_exceptions=False, + input="hello world", + ) + assert result.exit_code == 0, result.stdout + db = Database(db_path) + row = list(db["files"].rows)[0] + assert {"path": "stdin-name", "content": b"hello world", "size": 11} == row