diff --git a/docs/cli.rst b/docs/cli.rst index 7a127fe..a096fde 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -758,7 +758,7 @@ This also means you pipe ``sqlite-utils`` together to easily create a new SQLite Flattening nested JSON objects ------------------------------ -``sqlite-utils insert`` expects incoming data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. +``sqlite-utils insert`` and ``sqlite-utils memory`` both expect incoming JSON data to consist of an array of JSON objects, where the top-level keys of each object will become columns in the created database table. If your data is nested you can use the ``--flatten`` option to create columns that are derived from the nested data. diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index b29314e..cd0be82 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -1190,6 +1190,7 @@ def query( multiple=True, help="Additional databases to attach - specify alias and filepath", ) +@click.option("--flatten", is_flag=True, help="Flatten nested JSON objects") @output_options @click.option("-r", "--raw", is_flag=True, help="Raw output, first column of first row") @click.option( @@ -1226,6 +1227,7 @@ def memory( paths, sql, attach, + flatten, nl, arrays, csv, @@ -1300,6 +1302,8 @@ def memory( if format_used in (Format.CSV, Format.TSV) and not no_detect_types: tracker = TypeTracker() rows = tracker.wrap(rows) + if flatten: + rows = (dict(_flatten(row)) for row in rows) db[csv_table].insert_all(rows, alter=True) if tracker is not None: db[csv_table].transform(types=tracker.types) diff --git a/tests/test_cli_memory.py b/tests/test_cli_memory.py index a8b3a18..aee74e7 100644 --- a/tests/test_cli_memory.py +++ b/tests/test_cli_memory.py @@ -222,6 +222,30 @@ def test_memory_no_detect_types(option): ] +def test_memory_flatten(): + result = CliRunner().invoke( + cli.cli, + ["memory", "-", "select * from stdin", "--flatten"], + input=json.dumps( + { + "httpRequest": { + "latency": "0.112114537s", + "requestMethod": "GET", + }, + "insertId": "6111722f000b5b4c4d4071e2", + } + ), + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output.strip()) == [ + { + "httpRequest_latency": "0.112114537s", + "httpRequest_requestMethod": "GET", + "insertId": "6111722f000b5b4c4d4071e2", + } + ] + + def test_memory_analyze(): result = CliRunner().invoke( cli.cli,