From d4d075c92416e4cf3e835d79f25b410d370e7207 Mon Sep 17 00:00:00 2001 From: Parker Gurney Date: Wed, 5 Aug 2026 14:22:45 -0700 Subject: [PATCH] Add --extract option to insert/upsert commands Wires the extracts= Python API feature through the CLI so imports can create lookup tables in one step: --extract species for a table named after the column, or --extract species:Species for a custom name. Closes #352 --- docs/cli-reference.rst | 16 +++++++++++++ docs/cli.rst | 25 ++++++++++++++++++++ docs/python-api.rst | 2 ++ sqlite_utils/cli.py | 17 ++++++++++++++ tests/test_cli_insert.py | 51 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 39acf10..0b49fb9 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -317,6 +317,9 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr --alter Alter existing table to add any missing columns --not-null TEXT Columns that should be created as NOT NULL --default ... Default value that should be set for a column + --extract TEXT Extract this column into a separate lookup table, + e.g. --extract species or --extract species:Species + to use a custom table name --type ... Column types to use when creating the table --no-detect-types Treat all CSV/TSV columns as TEXT --analyze Run ANALYZE at the end of this operation @@ -384,6 +387,9 @@ See :ref:`cli_upsert`. --alter Alter existing table to add any missing columns --not-null TEXT Columns that should be created as NOT NULL --default ... Default value that should be set for a column + --extract TEXT Extract this column into a separate lookup table, + e.g. --extract species or --extract species:Species + to use a custom table name --type ... Column types to use when creating the table --no-detect-types Treat all CSV/TSV columns as TEXT --analyze Run ANALYZE at the end of this operation @@ -602,6 +608,12 @@ See :ref:`cli_insert_files`. -c size:size \ --pk name + Use --convert to transform each row before it is inserted, the same way as + sqlite-utils convert: + + sqlite-utils insert-files archive.db sqlar *.gif --sqlar \ + --convert 'row["data"] = zlib.compress(row["data"])' --import zlib + Options: -c, --column TEXT Column definitions for the table --pk TEXT Column to use as primary key @@ -610,7 +622,11 @@ See :ref:`cli_insert_files`. --upsert Upsert files with matching primary key --name TEXT File name to use --text Store file content as TEXT, not BLOB + --sqlar Store file content zlib-compressed, compatible with + SQLite's sqlar format --encoding TEXT Character encoding for input, defaults to utf-8 + --convert TEXT Python code to convert each row before insertion + --import TEXT Python modules to import -s, --silent Don't show a progress bar --load-extension TEXT Path to SQLite extension, with optional :entrypoint -h, --help Show this message and exit. diff --git a/docs/cli.rst b/docs/cli.rst index f74425a..b0757d5 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1445,6 +1445,31 @@ The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL`` or ` As with detected column types, ``--type`` only affects tables created by the command. If the table already exists, its existing column types are left unchanged. +.. _cli_insert_extract: + +Extracting columns into a separate table +----------------------------------------- + +Use ``--extract column-name`` to extract a column out into a separate lookup table during the insert, instead of running a separate :ref:`extract ` command afterwards. + +This can be used more than once, and works with both ``insert`` and ``upsert``: + +.. code-block:: bash + + sqlite-utils insert trees.db trees trees.csv --csv \ + --extract species + +This creates a ``species`` lookup table containing one row per distinct value, and replaces the ``species`` column on ``trees`` with a foreign key reference to it. + +To use a different name for the lookup table, add it after a colon: + +.. code-block:: bash + + sqlite-utils insert trees.db trees trees.csv --csv \ + --extract species:Species + +See :ref:`python_api_extracts` for more details on how this works, including how ``null`` values are handled. + To disable type detection and treat all columns as TEXT, use ``--no-detect-types``: .. code-block:: bash diff --git a/docs/python-api.rst b/docs/python-api.rst index d5a8ef8..db7b995 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1409,6 +1409,8 @@ To extract the ``species`` column out to a separate ``Species`` table, you can d ``None`` values are not extracted: no record is created for them in the lookup table and the column value stays ``null``. +The ``sqlite-utils insert`` and ``sqlite-utils upsert`` commands expose this as the ``--extract`` option, see :ref:`cli_insert_extract`. + .. _python_api_m2m: Working with many-to-many relationships diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 1118692..e777b09 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1026,6 +1026,14 @@ def insert_upsert_options(*, require_pk=False): type=(str, str), help="Default value that should be set for a column", ), + click.option( + "--extract", + "extract", + multiple=True, + help="Extract this column into a separate lookup table, e.g. " + "--extract species or --extract species:Species to use a " + "custom table name", + ), click.option( "--type", "types", @@ -1091,6 +1099,7 @@ def insert_upsert_implementation( truncate=False, not_null=None, default=None, + extract=None, types=None, no_detect_types=False, analyze=False, @@ -1119,6 +1128,10 @@ def insert_upsert_implementation( extra_kwargs["not_null"] = set(not_null) if default: extra_kwargs["defaults"] = dict(default) + if extract: + extra_kwargs["extracts"] = { + item.split(":", 1)[0]: item.split(":", 1)[-1] for item in extract + } if column_type_overrides: extra_kwargs["columns"] = column_type_overrides if upsert: @@ -1405,6 +1418,7 @@ def insert( truncate, not_null, default, + extract, types, strict, ): @@ -1500,6 +1514,7 @@ def insert( silent=silent, not_null=not_null, default=default, + extract=extract, types=types, strict=strict, code=code, @@ -1536,6 +1551,7 @@ def upsert( alter, not_null, default, + extract, types, no_detect_types, analyze, @@ -1588,6 +1604,7 @@ def upsert( key=key, not_null=not_null, default=default, + extract=extract, types=types, no_detect_types=no_detect_types, analyze=analyze, diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index 3289b27..f380a8a 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -212,6 +212,57 @@ def test_insert_not_null_default(db_path, tmpdir): ) == db["dogs"].schema +def test_insert_extract(db_path, tmpdir): + json_path = str(tmpdir / "trees.json") + trees = [ + {"id": 1, "name": "Lila", "species": "Oak"}, + {"id": 2, "name": "Suna", "species": "Oak"}, + {"id": 3, "name": "Pancake", "species": "Palm"}, + ] + with open(json_path, "w") as fp: + fp.write(json.dumps(trees)) + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "trees", json_path, "--pk", "id", "--extract", "species"], + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert {"trees", "species"} <= set(db.table_names()) + assert list(db["species"].rows) == [ + {"id": 1, "value": "Oak"}, + {"id": 2, "value": "Palm"}, + ] + assert list(db["trees"].rows) == [ + {"id": 1, "name": "Lila", "species": 1}, + {"id": 2, "name": "Suna", "species": 1}, + {"id": 3, "name": "Pancake", "species": 2}, + ] + + +def test_insert_extract_custom_table_name(db_path, tmpdir): + json_path = str(tmpdir / "trees.json") + trees = [{"id": 1, "species": "Oak"}] + with open(json_path, "w") as fp: + fp.write(json.dumps(trees)) + result = CliRunner().invoke( + cli.cli, + [ + "insert", + db_path, + "trees", + json_path, + "--pk", + "id", + "--extract", + "species:Species", + ], + ) + assert result.exit_code == 0, result.output + db = Database(db_path) + assert {"trees", "Species"} <= set(db.table_names()) + assert list(db["Species"].rows) == [{"id": 1, "value": "Oak"}] + + def test_insert_binary_base64(db_path): result = CliRunner().invoke( cli.cli,