remote/sqlite-utils
1
0
Fork 0
mirror of https://github.com/simonw/sqlite-utils.git synced 2026-09-07 17:14:09 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add --extract option to insert/upsert commands

Browse source 
Wires the extracts= Python API feature through the CLI so imports can
create lookup tables in one step: --extract species for a table named
after the column, or --extract species:Species for a custom name.

Closes #352
This commit is contained in:
Parker Gurney 2026-08-05 14:22:45 -07:00
commit d4d075c924
5 changed files with 111 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

16
docs/cli-reference.rst
View file

@ -317,6 +317,9 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr
--alter Alter existing table to add any missing columns
--not-null TEXT Columns that should be created as NOT NULL
--default <TEXT TEXT>... Default value that should be set for a column
--extract TEXT Extract this column into a separate lookup table,
e.g. --extract species or --extract species:Species
to use a custom table name
--type <TEXT CHOICE>... Column types to use when creating the table
--no-detect-types Treat all CSV/TSV columns as TEXT
--analyze Run ANALYZE at the end of this operation
@ -384,6 +387,9 @@ See :ref:`cli_upsert`.
--alter Alter existing table to add any missing columns
--not-null TEXT Columns that should be created as NOT NULL
--default <TEXT TEXT>... Default value that should be set for a column
--extract TEXT Extract this column into a separate lookup table,
e.g. --extract species or --extract species:Species
to use a custom table name
--type <TEXT CHOICE>... Column types to use when creating the table
--no-detect-types Treat all CSV/TSV columns as TEXT
--analyze Run ANALYZE at the end of this operation
@ -602,6 +608,12 @@ See :ref:`cli_insert_files`.
-c size:size \
--pk name
Use --convert to transform each row before it is inserted, the same way as
sqlite-utils convert:
sqlite-utils insert-files archive.db sqlar *.gif --sqlar \
--convert 'row["data"] = zlib.compress(row["data"])' --import zlib
Options:
-c, --column TEXT Column definitions for the table
--pk TEXT Column to use as primary key
@ -610,7 +622,11 @@ See :ref:`cli_insert_files`.
--upsert Upsert files with matching primary key
--name TEXT File name to use
--text Store file content as TEXT, not BLOB
--sqlar Store file content zlib-compressed, compatible with
SQLite's sqlar format
--encoding TEXT Character encoding for input, defaults to utf-8
--convert TEXT Python code to convert each row before insertion
--import TEXT Python modules to import
-s, --silent Don't show a progress bar
--load-extension TEXT Path to SQLite extension, with optional :entrypoint
-h, --help Show this message and exit.

25
docs/cli.rst
View file

@ -1445,6 +1445,31 @@ The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL`` or `
As with detected column types, ``--type`` only affects tables created by the command. If the table already exists, its existing column types are left unchanged.
.. _cli_insert_extract:
Extracting columns into a separate table
-----------------------------------------
Use ``--extract column-name`` to extract a column out into a separate lookup table during the insert, instead of running a separate :ref:`extract <cli_extract>` command afterwards.
This can be used more than once, and works with both ``insert`` and ``upsert``:
.. code-block:: bash
sqlite-utils insert trees.db trees trees.csv --csv \
--extract species
This creates a ``species`` lookup table containing one row per distinct value, and replaces the ``species`` column on ``trees`` with a foreign key reference to it.
To use a different name for the lookup table, add it after a colon:
.. code-block:: bash
sqlite-utils insert trees.db trees trees.csv --csv \
--extract species:Species
See :ref:`python_api_extracts` for more details on how this works, including how ``null`` values are handled.
To disable type detection and treat all columns as TEXT, use ``--no-detect-types``:
.. code-block:: bash

2
docs/python-api.rst
View file

@ -1409,6 +1409,8 @@ To extract the ``species`` column out to a separate ``Species`` table, you can d
``None`` values are not extracted: no record is created for them in the lookup table and the column value stays ``null``.
The ``sqlite-utils insert`` and ``sqlite-utils upsert`` commands expose this as the ``--extract`` option, see :ref:`cli_insert_extract`.
.. _python_api_m2m:
Working with many-to-many relationships

17
sqlite_utils/cli.py
View file

@ -1026,6 +1026,14 @@ def insert_upsert_options(*, require_pk=False):
type=(str, str),
help="Default value that should be set for a column",
),
click.option(
"--extract",
"extract",
multiple=True,
help="Extract this column into a separate lookup table, e.g. "
"--extract species or --extract species:Species to use a "
"custom table name",
),
click.option(
"--type",
"types",
@ -1091,6 +1099,7 @@ def insert_upsert_implementation(
truncate=False,
not_null=None,
default=None,
extract=None,
types=None,
no_detect_types=False,
analyze=False,
@ -1119,6 +1128,10 @@ def insert_upsert_implementation(
extra_kwargs["not_null"] = set(not_null)
if default:
extra_kwargs["defaults"] = dict(default)
if extract:
extra_kwargs["extracts"] = {
item.split(":", 1)[0]: item.split(":", 1)[-1] for item in extract
}
if column_type_overrides:
extra_kwargs["columns"] = column_type_overrides
if upsert:
@ -1405,6 +1418,7 @@ def insert(
truncate,
not_null,
default,
extract,
types,
strict,
):
@ -1500,6 +1514,7 @@ def insert(
silent=silent,
not_null=not_null,
default=default,
extract=extract,
types=types,
strict=strict,
code=code,
@ -1536,6 +1551,7 @@ def upsert(
alter,
not_null,
default,
extract,
types,
no_detect_types,
analyze,
@ -1588,6 +1604,7 @@ def upsert(
key=key,
not_null=not_null,
default=default,
extract=extract,
types=types,
no_detect_types=no_detect_types,
analyze=analyze,

51
tests/test_cli_insert.py
View file

@ -212,6 +212,57 @@ def test_insert_not_null_default(db_path, tmpdir):
) == db["dogs"].schema
def test_insert_extract(db_path, tmpdir):
json_path = str(tmpdir / "trees.json")
trees = [
{"id": 1, "name": "Lila", "species": "Oak"},
{"id": 2, "name": "Suna", "species": "Oak"},
{"id": 3, "name": "Pancake", "species": "Palm"},
]
with open(json_path, "w") as fp:
fp.write(json.dumps(trees))
result = CliRunner().invoke(
cli.cli,
["insert", db_path, "trees", json_path, "--pk", "id", "--extract", "species"],
)
assert result.exit_code == 0, result.output
db = Database(db_path)
assert {"trees", "species"} <= set(db.table_names())
assert list(db["species"].rows) == [
{"id": 1, "value": "Oak"},
{"id": 2, "value": "Palm"},
]
assert list(db["trees"].rows) == [
{"id": 1, "name": "Lila", "species": 1},
{"id": 2, "name": "Suna", "species": 1},
{"id": 3, "name": "Pancake", "species": 2},
]
def test_insert_extract_custom_table_name(db_path, tmpdir):
json_path = str(tmpdir / "trees.json")
trees = [{"id": 1, "species": "Oak"}]
with open(json_path, "w") as fp:
fp.write(json.dumps(trees))
result = CliRunner().invoke(
cli.cli,
[
"insert",
db_path,
"trees",
json_path,
"--pk",
"id",
"--extract",
"species:Species",
],
)
assert result.exit_code == 0, result.output
db = Database(db_path)
assert {"trees", "Species"} <= set(db.table_names())
assert list(db["Species"].rows) == [{"id": 1, "value": "Oak"}]
def test_insert_binary_base64(db_path):
result = CliRunner().invoke(
cli.cli,