diff --git a/docs/cli.rst b/docs/cli.rst index 8fdbbf1..0857d87 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -55,6 +55,40 @@ If you want to pretty-print the output further, you can pipe it through ``python } ] +.. _cli_json_values: + +Nested JSON values +------------------ + +If one of your columns contains JSON, by default it will be returned as an escaped string:: + + $ sqlite-utils dogs.db "select * from dogs" | python -mjson.tool + [ + { + "id": 1, + "name": "Cleo", + "friends": "[{\"name\": \"Pancakes\"}, {\"name\": \"Bailey\"}]" + } + ] + +You can use the ``--json-cols`` option to automatically detect these JSON columns and output them as nested JSON data:: + + $ sqlite-utils dogs.db "select * from dogs" --json-cols | python -mjson.tool + [ + { + "id": 1, + "name": "Cleo", + "friends": [ + { + "name": "Pancakes" + }, + { + "name": "Bailey" + } + ] + } + ] + .. _cli_query_csv: Running queries and returning CSV diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index fb07ee9..3d1c9a6 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -36,6 +36,12 @@ def output_options(fn): ), default="simple", ), + click.option( + "--json-cols", + help="Detect JSON cols and output them as JSON, not escaped strings", + is_flag=True, + default=False, + ), ) ): fn = decorator(fn) @@ -71,7 +77,20 @@ def cli(): is_flag=True, default=False, ) -def tables(path, fts4, fts5, counts, nl, arrays, csv, no_headers, table, fmt, columns): +def tables( + path, + fts4, + fts5, + counts, + nl, + arrays, + csv, + no_headers, + table, + fmt, + columns, + json_cols, +): """List the tables in the database""" db = sqlite_utils.Database(path) headers = ["table"] @@ -102,7 +121,7 @@ def tables(path, fts4, fts5, counts, nl, arrays, csv, no_headers, table, fmt, co for row in _iter(): writer.writerow(row) else: - for line in output_rows(_iter(), headers, nl, arrays): + for line in output_rows(_iter(), headers, nl, arrays, json_cols): click.echo(line) @@ -213,12 +232,8 @@ def create_index(path, table, column, name, unique, if_not_exists): ) @click.argument("table") @click.argument("column", nargs=-1, required=True) -@click.option( - "--fts4", help="Use FTS4", default=False, is_flag=True -) -@click.option( - "--fts5", help="Use FTS5", default=False, is_flag=True -) +@click.option("--fts4", help="Use FTS4", default=False, is_flag=True) +@click.option("--fts5", help="Use FTS5", default=False, is_flag=True) def enable_fts(path, table, column, fts4, fts5): "Enable FTS for specific table and columns" fts_version = "FTS5" @@ -332,7 +347,7 @@ def upsert(path, table, json_file, pk, nl, csv, batch_size, alter): ) @click.argument("sql") @output_options -def query(path, sql, nl, arrays, csv, no_headers, table, fmt): +def query(path, sql, nl, arrays, csv, no_headers, table, fmt, json_cols): "Execute SQL query and return the results as JSON" db = sqlite_utils.Database(path) cursor = iter(db.conn.execute(sql)) @@ -346,7 +361,7 @@ def query(path, sql, nl, arrays, csv, no_headers, table, fmt): for row in cursor: writer.writerow(row) else: - for line in output_rows(cursor, headers, nl, arrays): + for line in output_rows(cursor, headers, nl, arrays, json_cols): click.echo(line) @@ -359,7 +374,7 @@ def query(path, sql, nl, arrays, csv, no_headers, table, fmt): @click.argument("dbtable") @output_options @click.pass_context -def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt): +def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt, json_cols): "Output all rows in the specified table" ctx.invoke( query, @@ -374,7 +389,7 @@ def rows(ctx, path, dbtable, nl, arrays, csv, no_headers, table, fmt): ) -def output_rows(iterator, headers, nl, arrays): +def output_rows(iterator, headers, nl, arrays, json_cols): # We have to iterate two-at-a-time so we can know if we # should output a trailing comma or if we have reached # the last row. @@ -384,8 +399,11 @@ def output_rows(iterator, headers, nl, arrays): for row, next_row in itertools.zip_longest(current_iter, next_iter): is_last = next_row is None data = row + if json_cols: + # Any value that is a valid JSON string should be treated as JSON + data = [maybe_json(value) for value in data] if not arrays: - data = dict(zip(headers, row)) + data = dict(zip(headers, data)) line = "{firstchar}{serialized}{maybecomma}{lastchar}".format( firstchar=("[" if first else " ") if not nl else "", serialized=json.dumps(data), @@ -394,3 +412,15 @@ def output_rows(iterator, headers, nl, arrays): ) yield line first = False + + +def maybe_json(value): + if not isinstance(value, str): + return value + stripped = value.strip() + if not (stripped.startswith("{") or stripped.startswith("[")): + return value + try: + return json.loads(stripped) + except ValueError: + return value diff --git a/tests/test_cli.py b/tests/test_cli.py index 92bfcf3..c8185b7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -473,6 +473,37 @@ def test_query_json(db_path, sql, args, expected): assert expected == result.output.strip() +def test_query_json_with_json_cols(db_path): + db = Database(db_path) + with db.conn: + db["dogs"].insert( + { + "id": 1, + "name": "Cleo", + "friends": [{"name": "Pancakes"}, {"name": "Bailey"}], + } + ) + result = CliRunner().invoke( + cli.cli, [db_path, "select id, name, friends from dogs"] + ) + assert ( + r""" + [{"id": 1, "name": "Cleo", "friends": "[{\"name\": \"Pancakes\"}, {\"name\": \"Bailey\"}]"}] + """.strip() + == result.output.strip() + ) + # With --json-cols: + result = CliRunner().invoke( + cli.cli, [db_path, "select id, name, friends from dogs", "--json-cols"] + ) + assert ( + r""" + [{"id": 1, "name": "Cleo", "friends": [{"name": "Pancakes"}, {"name": "Bailey"}]}] + """.strip() + == result.output.strip() + ) + + @pytest.mark.parametrize( "args,expected", [