Add --extract option to insert/upsert commands

Wires the extracts= Python API feature through the CLI so imports can
create lookup tables in one step: --extract species for a table named
after the column, or --extract species:Species for a custom name.

Closes #352

docs/cli-reference.rst

@@ -317,6 +317,9 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr
       --alter                   Alter existing table to add any missing columns
       --not-null TEXT           Columns that should be created as NOT NULL
       --default <TEXT TEXT>...  Default value that should be set for a column
+      --extract TEXT            Extract this column into a separate lookup table,
+                                e.g. --extract species or --extract species:Species
+                                to use a custom table name
       --type <TEXT CHOICE>...   Column types to use when creating the table
       --no-detect-types         Treat all CSV/TSV columns as TEXT
       --analyze                 Run ANALYZE at the end of this operation
@@ -384,6 +387,9 @@ See :ref:`cli_upsert`.
       --alter                   Alter existing table to add any missing columns
       --not-null TEXT           Columns that should be created as NOT NULL
       --default <TEXT TEXT>...  Default value that should be set for a column
+      --extract TEXT            Extract this column into a separate lookup table,
+                                e.g. --extract species or --extract species:Species
+                                to use a custom table name
       --type <TEXT CHOICE>...   Column types to use when creating the table
       --no-detect-types         Treat all CSV/TSV columns as TEXT
       --analyze                 Run ANALYZE at the end of this operation
@@ -602,6 +608,12 @@ See :ref:`cli_insert_files`.
               -c size:size \
               --pk name
 
+      Use --convert to transform each row before it is inserted, the same way as
+      sqlite-utils convert:
+
+          sqlite-utils insert-files archive.db sqlar *.gif --sqlar \
+              --convert 'row["data"] = zlib.compress(row["data"])' --import zlib
+
     Options:
       -c, --column TEXT      Column definitions for the table
       --pk TEXT              Column to use as primary key
@@ -610,7 +622,11 @@ See :ref:`cli_insert_files`.
       --upsert               Upsert files with matching primary key
       --name TEXT            File name to use
       --text                 Store file content as TEXT, not BLOB
+      --sqlar                Store file content zlib-compressed, compatible with
+                             SQLite's sqlar format
       --encoding TEXT        Character encoding for input, defaults to utf-8
+      --convert TEXT         Python code to convert each row before insertion
+      --import TEXT          Python modules to import
       -s, --silent           Don't show a progress bar
       --load-extension TEXT  Path to SQLite extension, with optional :entrypoint
       -h, --help             Show this message and exit.

docs/cli.rst

@@ -1445,6 +1445,31 @@ The column type should be one of ``TEXT``, ``INTEGER``, ``FLOAT``, ``REAL`` or `
 
 As with detected column types, ``--type`` only affects tables created by the command. If the table already exists, its existing column types are left unchanged.
 
+.. _cli_insert_extract:
+
+Extracting columns into a separate table
+-----------------------------------------
+
+Use ``--extract column-name`` to extract a column out into a separate lookup table during the insert, instead of running a separate :ref:`extract <cli_extract>` command afterwards.
+
+This can be used more than once, and works with both ``insert`` and ``upsert``:
+
+.. code-block:: bash
+
+    sqlite-utils insert trees.db trees trees.csv --csv \
+        --extract species
+
+This creates a ``species`` lookup table containing one row per distinct value, and replaces the ``species`` column on ``trees`` with a foreign key reference to it.
+
+To use a different name for the lookup table, add it after a colon:
+
+.. code-block:: bash
+
+    sqlite-utils insert trees.db trees trees.csv --csv \
+        --extract species:Species
+
+See :ref:`python_api_extracts` for more details on how this works, including how ``null`` values are handled.
+
 To disable type detection and treat all columns as TEXT, use ``--no-detect-types``:
 
 .. code-block:: bash

docs/python-api.rst

@@ -1409,6 +1409,8 @@ To extract the ``species`` column out to a separate ``Species`` table, you can d
 
 ``None`` values are not extracted: no record is created for them in the lookup table and the column value stays ``null``.
 
+The ``sqlite-utils insert`` and ``sqlite-utils upsert`` commands expose this as the ``--extract`` option, see :ref:`cli_insert_extract`.
+
 .. _python_api_m2m:
 
 Working with many-to-many relationships

sqlite_utils/cli.py

@@ -1026,6 +1026,14 @@ def insert_upsert_options(*, require_pk=False):
                     type=(str, str),
                     help="Default value that should be set for a column",
                 ),
+                click.option(
+                    "--extract",
+                    "extract",
+                    multiple=True,
+                    help="Extract this column into a separate lookup table, e.g. "
+                    "--extract species or --extract species:Species to use a "
+                    "custom table name",
+                ),
                 click.option(
                     "--type",
                     "types",
@@ -1091,6 +1099,7 @@ def insert_upsert_implementation(
     truncate=False,
     not_null=None,
     default=None,
+    extract=None,
     types=None,
     no_detect_types=False,
     analyze=False,
@@ -1119,6 +1128,10 @@ def insert_upsert_implementation(
             extra_kwargs["not_null"] = set(not_null)
         if default:
             extra_kwargs["defaults"] = dict(default)
+        if extract:
+            extra_kwargs["extracts"] = {
+                item.split(":", 1)[0]: item.split(":", 1)[-1] for item in extract
+            }
         if column_type_overrides:
             extra_kwargs["columns"] = column_type_overrides
         if upsert:
@@ -1405,6 +1418,7 @@ def insert(
     truncate,
     not_null,
     default,
+    extract,
     types,
     strict,
 ):
@@ -1500,6 +1514,7 @@ def insert(
             silent=silent,
             not_null=not_null,
             default=default,
+            extract=extract,
             types=types,
             strict=strict,
             code=code,
@@ -1536,6 +1551,7 @@ def upsert(
     alter,
     not_null,
     default,
+    extract,
     types,
     no_detect_types,
     analyze,
@@ -1588,6 +1604,7 @@ def upsert(
             key=key,
             not_null=not_null,
             default=default,
+            extract=extract,
             types=types,
             no_detect_types=no_detect_types,
             analyze=analyze,

tests/test_cli_insert.py

@@ -212,6 +212,57 @@ def test_insert_not_null_default(db_path, tmpdir):
     ) == db["dogs"].schema
 
 
+def test_insert_extract(db_path, tmpdir):
+    json_path = str(tmpdir / "trees.json")
+    trees = [
+        {"id": 1, "name": "Lila", "species": "Oak"},
+        {"id": 2, "name": "Suna", "species": "Oak"},
+        {"id": 3, "name": "Pancake", "species": "Palm"},
+    ]
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps(trees))
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "trees", json_path, "--pk", "id", "--extract", "species"],
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert {"trees", "species"} <= set(db.table_names())
+    assert list(db["species"].rows) == [
+        {"id": 1, "value": "Oak"},
+        {"id": 2, "value": "Palm"},
+    ]
+    assert list(db["trees"].rows) == [
+        {"id": 1, "name": "Lila", "species": 1},
+        {"id": 2, "name": "Suna", "species": 1},
+        {"id": 3, "name": "Pancake", "species": 2},
+    ]
+
+
+def test_insert_extract_custom_table_name(db_path, tmpdir):
+    json_path = str(tmpdir / "trees.json")
+    trees = [{"id": 1, "species": "Oak"}]
+    with open(json_path, "w") as fp:
+        fp.write(json.dumps(trees))
+    result = CliRunner().invoke(
+        cli.cli,
+        [
+            "insert",
+            db_path,
+            "trees",
+            json_path,
+            "--pk",
+            "id",
+            "--extract",
+            "species:Species",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    db = Database(db_path)
+    assert {"trees", "Species"} <= set(db.table_names())
+    assert list(db["Species"].rows) == [{"id": 1, "value": "Oak"}]
+
+
 def test_insert_binary_base64(db_path):
     result = CliRunner().invoke(
         cli.cli,