Add --key option to insert/upsert/bulk for importing JSON lists nested under a top-level key

Fixes #489. JSON shaped like {"List": [...]} previously required piping through jq .List first; --key lets insert/upsert/bulk select the array directly.

docs/cli-reference.rst

@@ -293,6 +293,9 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr
       --flatten                 Flatten nested JSON objects, so {"a": {"b": 1}}
                                 becomes {"a_b": 1}
       --nl                      Expect newline-delimited JSON
+      --key TEXT                If input is a JSON object, import the array of
+                                records under this key, e.g. --key results for
+                                {"results": [...]}
       -c, --csv                 Expect CSV input
       --tsv                     Expect TSV input
       --empty-null              Treat empty strings as NULL
@@ -357,6 +360,9 @@ See :ref:`cli_upsert`.
       --flatten                 Flatten nested JSON objects, so {"a": {"b": 1}}
                                 becomes {"a_b": 1}
       --nl                      Expect newline-delimited JSON
+      --key TEXT                If input is a JSON object, import the array of
+                                records under this key, e.g. --key results for
+                                {"results": [...]}
       -c, --csv                 Expect CSV input
       --tsv                     Expect TSV input
       --empty-null              Treat empty strings as NULL
@@ -412,6 +418,9 @@ See :ref:`cli_bulk`.
       --flatten              Flatten nested JSON objects, so {"a": {"b": 1}} becomes
                              {"a_b": 1}
       --nl                   Expect newline-delimited JSON
+      --key TEXT             If input is a JSON object, import the array of records
+                             under this key, e.g. --key results for {"results":
+                             [...]}
       -c, --csv              Expect CSV input
       --tsv                  Expect TSV input
       --empty-null           Treat empty strings as NULL

docs/cli.rst

@@ -1200,6 +1200,24 @@ You can add the ``--analyze`` option to run ``ANALYZE`` against the table after
 .. note::
     In Python: :ref:`table.insert_all() <python_api_bulk_inserts>`  CLI reference: :ref:`sqlite-utils insert <cli_ref_insert>`
 
+If your JSON is a single object with the list of records nested under a key, use ``--key`` to select it.
+For example, if ``dogs.json`` looks like this:
+
+.. code-block:: json
+
+    {
+        "List": [
+            {"id": 1, "name": "Cleo"},
+            {"id": 2, "name": "Pancakes"}
+        ]
+    }
+
+You can import the records under the ``List`` key like so:
+
+.. code-block:: bash
+
+    sqlite-utils insert dogs.db dogs dogs.json --key List
+
 .. _cli_inserting_data_binary:
 
 Inserting binary data

sqlite_utils/cli.py

@@ -928,6 +928,10 @@ _import_options = (
         help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}',
     ),
     click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"),
+    click.option(
+        "--key",
+        help='If input is a JSON object, import the array of records under this key, e.g. --key results for {"results": [...]}',
+    ),
     click.option("-c", "--csv", is_flag=True, help="Expect CSV input"),
     click.option("--tsv", is_flag=True, help="Expect TSV input"),
     click.option("--empty-null", is_flag=True, help="Treat empty strings as NULL"),
@@ -1071,6 +1075,7 @@ def insert_upsert_implementation(
     stop_after,
     alter,
     upsert,
+    key=None,
     ignore=False,
     replace=False,
     truncate=False,
@@ -1186,6 +1191,7 @@ def insert_upsert_implementation(
                 delimiter,
                 quotechar,
                 encoding,
+                key,
             ]
         ):
             raise click.ClickException(
@@ -1203,6 +1209,10 @@ def insert_upsert_implementation(
         csv = True
     if (nl + csv + tsv) >= 2:
         raise click.ClickException("Use just one of --nl, --csv or --tsv")
+    if key and (nl or csv or tsv or lines or text):
+        raise click.ClickException(
+            "--key cannot be used with --nl, --csv, --tsv, --lines or --text"
+        )
     if (csv or tsv) and flatten:
         raise click.ClickException("--flatten cannot be used with --csv or --tsv")
     if empty_null and not (csv or tsv):
@@ -1267,7 +1277,17 @@ def insert_upsert_implementation(
                     docs = (json.loads(line) for line in decoded if line.strip())
                 else:
                     docs = json.load(decoded)
-                    if isinstance(docs, dict):
+                    if key:
+                        if not isinstance(docs, dict) or key not in docs:
+                            raise click.ClickException(
+                                f"JSON key '{key}' not found in input"
+                            )
+                        docs = docs[key]
+                        if not isinstance(docs, list):
+                            raise click.ClickException(
+                                f"JSON key '{key}' is not a list"
+                            )
+                    elif isinstance(docs, dict):
                         docs = [docs]
             except json.decoder.JSONDecodeError as ex:
                 raise click.ClickException(
@@ -1350,6 +1370,7 @@ def insert(
     code,
     flatten,
     nl,
+    key,
     csv,
     tsv,
     empty_null,
@@ -1459,6 +1480,7 @@ def insert(
             stop_after,
             alter=alter,
             upsert=False,
+            key=key,
             ignore=ignore,
             replace=replace,
             truncate=truncate,
@@ -1486,6 +1508,7 @@ def upsert(
     code,
     flatten,
     nl,
+    key,
     csv,
     tsv,
     empty_null,
@@ -1552,6 +1575,7 @@ def upsert(
             stop_after,
             alter=alter,
             upsert=True,
+            key=key,
             not_null=not_null,
             default=default,
             types=types,
@@ -1586,6 +1610,7 @@ def bulk(
     functions,
     flatten,
     nl,
+    key,
     csv,
     tsv,
     empty_null,
@@ -1621,6 +1646,7 @@ def bulk(
             pk=None,
             flatten=flatten,
             nl=nl,
+            key=key,
             csv=csv,
             tsv=tsv,
             empty_null=empty_null,

tests/test_cli_insert.py

@@ -79,6 +79,55 @@ def test_insert_json_flatten_nl(tmpdir):
     ]
 
 
+def test_insert_json_key(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "dogs", "-", "--key", "List"],
+        input=json.dumps(
+            {"List": [{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Suna"}]}
+        ),
+    )
+    assert result.exit_code == 0
+    assert list(Database(db_path).query("select * from dogs")) == [
+        {"id": 1, "name": "Cleo"},
+        {"id": 2, "name": "Suna"},
+    ]
+
+
+def test_insert_json_key_missing_errors(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "dogs", "-", "--key", "Missing"],
+        input=json.dumps({"List": [{"id": 1}]}),
+    )
+    assert result.exit_code == 1
+    assert "JSON key 'Missing' not found in input" in result.output
+
+
+def test_insert_json_key_not_a_list_errors(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "dogs", "-", "--key", "List"],
+        input=json.dumps({"List": {"id": 1}}),
+    )
+    assert result.exit_code == 1
+    assert "JSON key 'List' is not a list" in result.output
+
+
+def test_insert_json_key_rejects_nl(tmpdir):
+    db_path = str(tmpdir / "dogs.db")
+    result = CliRunner().invoke(
+        cli.cli,
+        ["insert", db_path, "dogs", "-", "--key", "List", "--nl"],
+        input=json.dumps({"id": 1}),
+    )
+    assert result.exit_code == 1
+    assert "--key cannot be used with" in result.output
+
+
 @pytest.mark.parametrize(
     "args,expected_pks",
     (