From 6a2e9418632bcb9444b060aec29610d458deeace Mon Sep 17 00:00:00 2001 From: Parker Gurney Date: Wed, 5 Aug 2026 14:10:18 -0700 Subject: [PATCH] Add --key option to insert/upsert/bulk for importing JSON lists nested under a top-level key Fixes #489. JSON shaped like {"List": [...]} previously required piping through jq .List first; --key lets insert/upsert/bulk select the array directly. --- docs/cli-reference.rst | 9 ++++++++ docs/cli.rst | 18 +++++++++++++++ sqlite_utils/cli.py | 28 ++++++++++++++++++++++- tests/test_cli_insert.py | 49 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 1 deletion(-) diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index a4ec402..0a27438 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -293,6 +293,9 @@ See :ref:`cli_inserting_data`, :ref:`cli_insert_csv_tsv`, :ref:`cli_insert_unstr --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON + --key TEXT If input is a JSON object, import the array of + records under this key, e.g. --key results for + {"results": [...]} -c, --csv Expect CSV input --tsv Expect TSV input --empty-null Treat empty strings as NULL @@ -357,6 +360,9 @@ See :ref:`cli_upsert`. --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON + --key TEXT If input is a JSON object, import the array of + records under this key, e.g. --key results for + {"results": [...]} -c, --csv Expect CSV input --tsv Expect TSV input --empty-null Treat empty strings as NULL @@ -412,6 +418,9 @@ See :ref:`cli_bulk`. --flatten Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1} --nl Expect newline-delimited JSON + --key TEXT If input is a JSON object, import the array of records + under this key, e.g. --key results for {"results": + [...]} -c, --csv Expect CSV input --tsv Expect TSV input --empty-null Treat empty strings as NULL diff --git a/docs/cli.rst b/docs/cli.rst index a4b88ce..d896a6e 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1200,6 +1200,24 @@ You can add the ``--analyze`` option to run ``ANALYZE`` against the table after .. note:: In Python: :ref:`table.insert_all() ` CLI reference: :ref:`sqlite-utils insert ` +If your JSON is a single object with the list of records nested under a key, use ``--key`` to select it. +For example, if ``dogs.json`` looks like this: + +.. code-block:: json + + { + "List": [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Pancakes"} + ] + } + +You can import the records under the ``List`` key like so: + +.. code-block:: bash + + sqlite-utils insert dogs.db dogs dogs.json --key List + .. _cli_inserting_data_binary: Inserting binary data diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index 169849d..a3c37fd 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -928,6 +928,10 @@ _import_options = ( help='Flatten nested JSON objects, so {"a": {"b": 1}} becomes {"a_b": 1}', ), click.option("--nl", is_flag=True, help="Expect newline-delimited JSON"), + click.option( + "--key", + help='If input is a JSON object, import the array of records under this key, e.g. --key results for {"results": [...]}', + ), click.option("-c", "--csv", is_flag=True, help="Expect CSV input"), click.option("--tsv", is_flag=True, help="Expect TSV input"), click.option("--empty-null", is_flag=True, help="Treat empty strings as NULL"), @@ -1071,6 +1075,7 @@ def insert_upsert_implementation( stop_after, alter, upsert, + key=None, ignore=False, replace=False, truncate=False, @@ -1186,6 +1191,7 @@ def insert_upsert_implementation( delimiter, quotechar, encoding, + key, ] ): raise click.ClickException( @@ -1203,6 +1209,10 @@ def insert_upsert_implementation( csv = True if (nl + csv + tsv) >= 2: raise click.ClickException("Use just one of --nl, --csv or --tsv") + if key and (nl or csv or tsv or lines or text): + raise click.ClickException( + "--key cannot be used with --nl, --csv, --tsv, --lines or --text" + ) if (csv or tsv) and flatten: raise click.ClickException("--flatten cannot be used with --csv or --tsv") if empty_null and not (csv or tsv): @@ -1267,7 +1277,17 @@ def insert_upsert_implementation( docs = (json.loads(line) for line in decoded if line.strip()) else: docs = json.load(decoded) - if isinstance(docs, dict): + if key: + if not isinstance(docs, dict) or key not in docs: + raise click.ClickException( + f"JSON key '{key}' not found in input" + ) + docs = docs[key] + if not isinstance(docs, list): + raise click.ClickException( + f"JSON key '{key}' is not a list" + ) + elif isinstance(docs, dict): docs = [docs] except json.decoder.JSONDecodeError as ex: raise click.ClickException( @@ -1350,6 +1370,7 @@ def insert( code, flatten, nl, + key, csv, tsv, empty_null, @@ -1459,6 +1480,7 @@ def insert( stop_after, alter=alter, upsert=False, + key=key, ignore=ignore, replace=replace, truncate=truncate, @@ -1486,6 +1508,7 @@ def upsert( code, flatten, nl, + key, csv, tsv, empty_null, @@ -1552,6 +1575,7 @@ def upsert( stop_after, alter=alter, upsert=True, + key=key, not_null=not_null, default=default, types=types, @@ -1586,6 +1610,7 @@ def bulk( functions, flatten, nl, + key, csv, tsv, empty_null, @@ -1621,6 +1646,7 @@ def bulk( pk=None, flatten=flatten, nl=nl, + key=key, csv=csv, tsv=tsv, empty_null=empty_null, diff --git a/tests/test_cli_insert.py b/tests/test_cli_insert.py index eefb3fa..3289b27 100644 --- a/tests/test_cli_insert.py +++ b/tests/test_cli_insert.py @@ -79,6 +79,55 @@ def test_insert_json_flatten_nl(tmpdir): ] +def test_insert_json_key(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", "-", "--key", "List"], + input=json.dumps( + {"List": [{"id": 1, "name": "Cleo"}, {"id": 2, "name": "Suna"}]} + ), + ) + assert result.exit_code == 0 + assert list(Database(db_path).query("select * from dogs")) == [ + {"id": 1, "name": "Cleo"}, + {"id": 2, "name": "Suna"}, + ] + + +def test_insert_json_key_missing_errors(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", "-", "--key", "Missing"], + input=json.dumps({"List": [{"id": 1}]}), + ) + assert result.exit_code == 1 + assert "JSON key 'Missing' not found in input" in result.output + + +def test_insert_json_key_not_a_list_errors(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", "-", "--key", "List"], + input=json.dumps({"List": {"id": 1}}), + ) + assert result.exit_code == 1 + assert "JSON key 'List' is not a list" in result.output + + +def test_insert_json_key_rejects_nl(tmpdir): + db_path = str(tmpdir / "dogs.db") + result = CliRunner().invoke( + cli.cli, + ["insert", db_path, "dogs", "-", "--key", "List", "--nl"], + input=json.dumps({"id": 1}), + ) + assert result.exit_code == 1 + assert "--key cannot be used with" in result.output + + @pytest.mark.parametrize( "args,expected_pks", (