From ebc802f7ff0e640b6ae11ea525290fea0115228c Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Mon, 27 Jul 2020 00:08:57 -0700 Subject: [PATCH] sqlite-utils insert-files command, closes #122 --- docs/cli.rst | 74 ++++++++++++++++++++++++++ sqlite_utils/cli.py | 106 +++++++++++++++++++++++++++++++++++++ tests/test_docs.py | 2 +- tests/test_insert_files.py | 85 +++++++++++++++++++++++++++++ 4 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 tests/test_insert_files.py diff --git a/docs/cli.rst b/docs/cli.rst index f3892ca..4050824 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -384,6 +384,80 @@ The command will fail if you reference columns that do not exist on the table. T .. note:: ``upsert`` in sqlite-utils 1.x worked like ``insert ... --replace`` does in 2.x. See `issue #66 `__ for details of this change. +.. _cli_insert_files: + +Inserting binary data from files +================================ + +SQLite ``BLOB`` columns can be used to store binary content. It can be useful to insert the contents of files into a SQLite table. + +The ``insert-files`` command can be used to insert the content of files, along with their metadata. + +Here's an example that inserts all of the GIF files in the current directory into a ``gifs.db`` database, placing the file contents in an ``images`` table:: + + $ sqlite-utils insert-files gifs.db images *.gif + +You can also pass one or more directories, in which case every file in those directories will be added recursively:: + + $ sqlite-utils insert-files gifs.db images path/to/my-gifs + +By default this command will create a table with the following schema:: + + CREATE TABLE [images] ( + [path] TEXT PRIMARY KEY, + [content] BLOB, + [size] INTEGER + ); + +You can customize the schema using one or more ``-c`` options. For a table schema that includes just the path, MD5 hash and last modification time of the file, you would use this:: + + $ sqlite-utils insert-files gifs.db images *.gif -c path -c content -c mtime --pk=path + +This will result in the following schema:: + + CREATE TABLE [images] ( + [path] TEXT PRIMARY KEY, + [content] BLOB, + [mtime] FLOAT + ); + +You can change the name of one of these columns using a ``-c colname:coldef`` parameter. To rename the ``mtime`` column to ``last_modified`` you would use this:: + + $ sqlite-utils insert-files gifs.db images *.gif \ + -c path -c content -c last_modified:mtime --pk=path + +You can pass ``--replace`` or ``--upsert`` to indicate what should happen if you try to insert a file with an existing primary key. Pass ``--alter`` to cause any missing columns to be added to the table. + +The full list of column definitions you can use is as follows: + +``name`` + The name of the file, e.g. ``cleo.jpg`` +``path`` + The path to the file relative to the root folder, e.g. ``pictures/cleo.jpg`` +``fullpath`` + The fully resolved path to the image, e.g. ``/home/simonw/pictures/cleo.jpg`` +``sha256`` + The SHA256 hash of the file contents +``md5`` + The MD5 hash of the file contents +``mode`` + The permission bits of the file, as an integer - you may want to convert this to octal +``content`` + The binary file contents, which will be stored as a BLOB +``mtime`` + The modification time of the file, as floating point seconds since the Unix epoch +``ctime`` + The creation time of the file, as floating point seconds since the Unix epoch +``mtime_int`` + The modification time as an integer rather than a float +``ctime_int`` + The creation time as an integer rather than a float +``mtime_iso`` + The modification time as an ISO timestamp, e.g. ``2020-07-27T04:24:06.654246`` +``ctime_iso`` + The creation time is an ISO timestamp +``size`` + The integer size of the file in bytes .. _cli_create_table: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 219c0bf..3dc40ea 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1,6 +1,9 @@ import base64 import click from click_default_group import DefaultGroup +from datetime import datetime +import hashlib +import pathlib import sqlite_utils from sqlite_utils.db import AlterError import itertools @@ -741,6 +744,109 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols) ) +@cli.command(name="insert-files") +@click.argument( + "path", + type=click.Path(file_okay=True, dir_okay=False, allow_dash=False), + required=True, +) +@click.argument("table") +@click.argument( + "file_or_dir", + nargs=-1, + required=True, + type=click.Path(file_okay=True, dir_okay=True, allow_dash=True), +) +@click.option( + "-c", "--column", type=str, multiple=True, help="Column definitions for the table", +) +@click.option("--pk", type=str, help="Column to use as primary key") +@click.option("--alter", is_flag=True, help="Alter table to add missing columns") +@click.option("--replace", is_flag=True, help="Replace files with matching primary key") +@click.option("--upsert", is_flag=True, help="Upsert files with matching primary key") +def insert_files(path, table, file_or_dir, column, pk, alter, replace, upsert): + """ + Insert one or more files using BLOB columns in the specified table + + Example usage: + + \b + sqlite-utils insert-files pics.db images *.gif \\ + -c name:name \\ + -c content:content \\ + -c content_hash:sha256 \\ + -c created:ctime_iso \\ + -c modified:mtime_iso \\ + -c size:size \\ + --pk name + """ + if not column: + column = ["path:path", "content:content", "size:size"] + if not pk: + pk = "path" + + def yield_paths_and_relative_paths(): + for f_or_d in file_or_dir: + path = pathlib.Path(f_or_d) + if path.is_dir(): + for subpath in path.rglob("*"): + if subpath.is_file(): + yield subpath, subpath.relative_to(path) + elif path.is_file(): + yield path, path + + # Load all paths so we can show a progress bar + paths_and_relative_paths = list(yield_paths_and_relative_paths()) + + with click.progressbar(paths_and_relative_paths) as bar: + + def to_insert(): + for path, relative_path in bar: + row = {} + for coldef in column: + if ":" in coldef: + colname, coltype = coldef.rsplit(":", 1) + else: + colname, coltype = coldef, coldef + try: + if coltype == "path": + value = str(relative_path) + else: + value = FILE_COLUMNS[coltype](path) + row[colname] = value + except KeyError: + raise click.ClickException( + "'{}' is not a valid column definition - options are {}".format( + coltype, ", ".join(FILE_COLUMNS.keys()) + ) + ) + yield row + + db = sqlite_utils.Database(path) + with db.conn: + db[table].insert_all( + to_insert(), pk=pk, alter=alter, replace=replace, upsert=upsert + ) + + +FILE_COLUMNS = { + "name": lambda p: p.name, + "path": lambda p: str(p), + "fullpath": lambda p: str(p.resolve()), + "sha256": lambda p: hashlib.sha256(p.resolve().read_bytes()).hexdigest(), + "md5": lambda p: hashlib.md5(p.resolve().read_bytes()).hexdigest(), + "mode": lambda p: p.stat().st_mode, + "content": lambda p: p.resolve().read_bytes(), + "mtime": lambda p: p.stat().st_mtime, + "ctime": lambda p: p.stat().st_ctime, + "mtime_int": lambda p: int(p.stat().st_mtime), + "ctime_int": lambda p: int(p.stat().st_ctime), + "mtime_iso": lambda p: datetime.utcfromtimestamp(p.stat().st_mtime).isoformat(), + "ctime_iso": lambda p: datetime.utcfromtimestamp(p.stat().st_ctime).isoformat(), + "size": lambda p: p.stat().st_size, +} + + def output_rows(iterator, headers, nl, arrays, json_cols): # We have to iterate two-at-a-time so we can know if we # should output a trailing comma or if we have reached diff --git a/tests/test_docs.py b/tests/test_docs.py index 61f0ece..3b3e4a5 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -18,5 +18,5 @@ def documented_commands(): @pytest.mark.parametrize("command", cli.cli.commands.keys()) -def test_plugin_hooks_are_documented(documented_commands, command): +def test_commands_are_documented(documented_commands, command): assert command in documented_commands diff --git a/tests/test_insert_files.py b/tests/test_insert_files.py new file mode 100644 index 0000000..7a0dd8a --- /dev/null +++ b/tests/test_insert_files.py @@ -0,0 +1,85 @@ +from sqlite_utils import cli, Database +from click.testing import CliRunner +import pathlib + + +def test_insert_files(): + runner = CliRunner() + with runner.isolated_filesystem(): + tmpdir = pathlib.Path(".") + print("tmpdir = ", tmpdir.resolve()) + db_path = str(tmpdir / "files.db") + (tmpdir / "one.txt").write_text("This is file one", "utf-8") + (tmpdir / "two.txt").write_text("Two is shorter", "utf-8") + (tmpdir / "nested").mkdir() + (tmpdir / "nested" / "three.txt").write_text("Three is nested", "utf-8") + coltypes = ( + "name", + "path", + "fullpath", + "sha256", + "md5", + "mode", + "content", + "mtime", + "ctime", + "mtime_int", + "ctime_int", + "mtime_iso", + "ctime_iso", + "size", + ) + cols = [] + for coltype in coltypes: + cols += ["-c", "{}:{}".format(coltype, coltype)] + result = runner.invoke( + cli.cli, + ["insert-files", db_path, "files", str(tmpdir)] + cols + ["--pk", "path"], + catch_exceptions=False, + ) + assert result.exit_code == 0, result.stdout + db = Database(db_path) + rows_by_path = {r["path"]: r for r in db["files"].rows} + one, two, three = ( + rows_by_path["one.txt"], + rows_by_path["two.txt"], + rows_by_path["nested/three.txt"], + ) + assert { + "content": b"This is file one", + "md5": "556dfb57fce9ca301f914e2273adf354", + "name": "one.txt", + "path": "one.txt", + "sha256": "e34138f26b5f7368f298b4e736fea0aad87ddec69fbd04dc183b20f4d844bad5", + "size": 16, + }.items() <= one.items() + assert { + "content": b"Two is shorter", + "md5": "f86f067b083af1911043eb215e74ac70", + "name": "two.txt", + "path": "two.txt", + "sha256": "9368988ed16d4a2da0af9db9b686d385b942cb3ffd4e013f43aed2ec041183d9", + "size": 14, + }.items() <= two.items() + assert { + "content": b"Three is nested", + "md5": "12580f341781f5a5b589164d3cd39523", + "name": "three.txt", + "path": "nested/three.txt", + "sha256": "6dd45aaaaa6b9f96af19363a92c8fca5d34791d3c35c44eb19468a6a862cc8cd", + "size": 15, + }.items() <= three.items() + # Assert the other int/str/float columns exist and are of the right types + expected_types = { + "ctime": float, + "ctime_int": int, + "ctime_iso": str, + "mtime": float, + "mtime_int": int, + "mtime_iso": str, + "mode": int, + "fullpath": str, + } + for colname, expected_type in expected_types.items(): + for row in (one, two, three): + assert isinstance(row[colname], expected_type)