From f34dfb25da28f088aa37a6064b200e8467e627fe Mon Sep 17 00:00:00 2001 From: Parker Gurney Date: Wed, 5 Aug 2026 14:18:25 -0700 Subject: [PATCH] Auto-decode columns declared as JSON in output modes (issue #579) Columns declared with type JSON now decode automatically wherever nested JSON is already supported (query/rows/search/memory output, and the Python API), without needing --json-cols. Uses sqlite3's built-in detect_types=PARSE_DECLTYPES + a registered JSON converter, so it works transparently through joins and aliases. Flat formats (CSV/TSV/table/raw) re-serialize decoded values back to JSON text instead of a Python repr. Also adds "JSON" as a first-class column type for create()/add_column()/ transform(), and fixes column_affinity("JSON") so transform() doesn't rewrite JSON columns as REAL. --- docs/cli.rst | 10 +++++++++- docs/python-api.rst | 3 ++- sqlite_utils/cli.py | 21 ++++++++++++++++----- sqlite_utils/db.py | 15 ++++++++++++--- sqlite_utils/utils.py | 17 +++++++++++++++++ tests/test_cli.py | 34 ++++++++++++++++++++++++++++++++++ tests/test_column_affinity.py | 3 +++ tests/test_create.py | 6 ++++++ 8 files changed, 99 insertions(+), 10 deletions(-) diff --git a/docs/cli.rst b/docs/cli.rst index a1f3f55..f74425a 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -192,7 +192,15 @@ If one of your columns contains JSON, by default it will be returned as an escap } ] -You can use the ``--json-cols`` option to automatically detect these JSON columns and output them as nested JSON data: +If the column is declared as type ``JSON`` in the table's schema, this happens automatically without needing ``--json-cols``: + +.. code-block:: bash + + sqlite-utils dogs.db "CREATE TABLE dogs2 (id integer primary key, name text, friends JSON)" + +Columns declared this way are decoded as nested JSON by every command that returns JSON (``rows``, ``query``, ``search``, ``memory``), and by the Python library too. CSV, TSV and table output always show the underlying text, since flat formats cannot represent nested structures. + +For other TEXT columns that merely happen to contain a JSON string, you can use the ``--json-cols`` option to automatically detect these JSON columns and output them as nested JSON data: .. code-block:: bash diff --git a/docs/python-api.rst b/docs/python-api.rst index c8750c9..d5a8ef8 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -1564,12 +1564,13 @@ You can add a new column to a table using the ``.add_column(col_name, col_type)` db.table("dogs").add_column("dob", datetime.date) db.table("dogs").add_column("image", "BLOB") db.table("dogs").add_column("website") # str by default + db.table("dogs").add_column("friends", "JSON") You can specify the ``col_type`` argument either using a SQLite type as a string, or by directly passing a Python type e.g. ``str`` or ``float``. The ``col_type`` is optional - if you omit it the type of ``TEXT`` will be used. -SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"`` or ``"BLOB"``. +SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"``, ``"BLOB"`` or ``"JSON"``. A column declared as ``"JSON"`` will be automatically decoded back into a Python object (rather than a JSON string) whenever it is read, both by the Python library and by the :ref:`CLI `. If you pass a Python type, it will be mapped to SQLite types as shown here:: diff --git a/sqlite_utils/cli.py b/sqlite_utils/cli.py index da04b83..1118692 100644 --- a/sqlite_utils/cli.py +++ b/sqlite_utils/cli.py @@ -2323,14 +2323,14 @@ def _execute_query( cursor_or_rows: Any = cursor if raw: row = cursor_or_rows.fetchone() - data = row[0] if row else None + data = _flatten_json(row[0]) if row else None if isinstance(data, bytes): sys.stdout.buffer.write(data) else: sys.stdout.write(str(data)) elif raw_lines: for row in cursor: - data = row[0] + data = _flatten_json(row[0]) if isinstance(data, bytes): sys.stdout.buffer.write(data + b"\n") else: @@ -2341,7 +2341,7 @@ def _execute_query( elif fmt or table: print( tabulate.tabulate( - list(cursor), + [[_flatten_json(value) for value in row] for row in cursor], headers=() if no_headers else headers, tablefmt=fmt or "simple", ) @@ -2351,7 +2351,7 @@ def _execute_query( if not no_headers: writer.writerow(headers) for row in cursor: - writer.writerow(row) + writer.writerow([_flatten_json(value) for value in row]) else: for line in output_rows(cursor, headers, nl, arrays, json_cols, ascii_): click.echo(line) @@ -3826,7 +3826,9 @@ def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False): def output_transpose(rows, headers, no_headers): # psql-style extended display: one "key = value" block per row headers = [str(h) for h in headers] - str_rows = [["" if v is None else str(v) for v in row] for row in rows] + str_rows = [ + ["" if v is None else str(_flatten_json(v)) for v in row] for row in rows + ] key_width = max([len(h) for h in headers], default=0) val_width = max([len(v) for row in str_rows for v in row], default=0) total_width = key_width + 3 + val_width @@ -3837,6 +3839,15 @@ def output_transpose(rows, headers, no_headers): yield "{} | {}".format(header.ljust(key_width), value) +def _flatten_json(value): + # Columns declared as JSON are auto-decoded to dict/list by the sqlite3 + # driver (see issue 579) - flat output formats (CSV/TSV/table/raw) need + # that turned back into a JSON string rather than a Python repr + if isinstance(value, (dict, list)): + return json.dumps(value) + return value + + def maybe_json(value): if not isinstance(value, str): return value diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index 713b110..acc0356 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -335,6 +335,7 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = { "FLOAT": "FLOAT", "REAL": "REAL", "BLOB": "BLOB", + "JSON": "JSON", "text": "TEXT", "str": "TEXT", "integer": "INTEGER", @@ -343,6 +344,7 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = { "real": "REAL", "blob": "BLOB", "bytes": "BLOB", + "json": "JSON", } # If numpy is available, add more types if np: @@ -532,11 +534,14 @@ class Database: uri, uri=True, check_same_thread=False, + detect_types=sqlite3.PARSE_DECLTYPES, ) self.memory = True self.memory_name = memory_name elif memory or filename_or_conn == ":memory:": - self.conn = sqlite3.connect(":memory:") + self.conn = sqlite3.connect( + ":memory:", detect_types=sqlite3.PARSE_DECLTYPES + ) self.memory = True elif isinstance(filename_or_conn, (str, pathlib.Path)): if recreate and os.path.exists(filename_or_conn): @@ -545,9 +550,13 @@ class Database: except OSError: # Avoid mypy and __repr__ errors, see: # https://github.com/simonw/sqlite-utils/issues/503 - self.conn = sqlite3.connect(":memory:") + self.conn = sqlite3.connect( + ":memory:", detect_types=sqlite3.PARSE_DECLTYPES + ) raise - self.conn = sqlite3.connect(str(filename_or_conn)) + self.conn = sqlite3.connect( + str(filename_or_conn), detect_types=sqlite3.PARSE_DECLTYPES + ) else: if recreate: raise ValueError("recreate cannot be used with connections, only paths") diff --git a/sqlite_utils/utils.py b/sqlite_utils/utils.py index abd23b1..3de6aee 100644 --- a/sqlite_utils/utils.py +++ b/sqlite_utils/utils.py @@ -170,6 +170,10 @@ def column_affinity(column_type: str) -> type: column_type = column_type.upper().strip() if column_type == "": return str # We differ from spec, which says it should be BLOB + if "JSON" in column_type: + # Not a real SQLite affinity, but sqlite-utils treats JSON as a + # supported column type (see issue 579) and it is stored as text + return str if "INT" in column_type: return int if "CHAR" in column_type or "CLOB" in column_type or "TEXT" in column_type: @@ -182,6 +186,19 @@ def column_affinity(column_type: str) -> type: return float +def _json_column_converter(raw: bytes) -> Any: + # Registered as the sqlite3 converter for columns declared as JSON, see + # https://github.com/simonw/sqlite-utils/issues/579 - falls back to the + # decoded text unchanged if it does not turn out to be valid JSON + try: + return json.loads(raw) + except ValueError: + return raw.decode("utf-8", "replace") + + +sqlite3.register_converter("JSON", _json_column_converter) + + def decode_base64_values(doc: dict[str, Any]) -> dict[str, Any]: # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them to_fix = [ diff --git a/tests/test_cli.py b/tests/test_cli.py index 3945d87..350ec61 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1137,6 +1137,40 @@ def test_query_json_with_json_cols(db_path): assert expected == result_rows.output.strip() +def test_query_json_declared_column_type_auto_decoded(db_path): + # Columns declared as JSON should be decoded automatically, without + # needing --json-cols - see https://github.com/simonw/sqlite-utils/issues/579 + db = Database(db_path) + with db.conn: + db.execute( + "CREATE TABLE dogs (id INTEGER PRIMARY KEY, name TEXT, friends JSON)" + ) + db["dogs"].insert( + { + "id": 1, + "name": "Cleo", + "friends": json.dumps([{"name": "Pancakes"}, {"name": "Bailey"}]), + } + ) + expected = r""" + [{"id": 1, "name": "Cleo", "friends": [{"name": "Pancakes"}, {"name": "Bailey"}]}] + """.strip() + result = CliRunner().invoke( + cli.cli, [db_path, "select id, name, friends from dogs"] + ) + assert expected == result.output.strip() + result_rows = CliRunner().invoke(cli.cli, ["rows", db_path, "dogs"]) + assert expected == result_rows.output.strip() + # Flat formats should keep it as a JSON string, not a Python repr + result_csv = CliRunner().invoke( + cli.cli, [db_path, "select id, name, friends from dogs", "--csv"] + ) + assert ( + '1,Cleo,"[{""name"": ""Pancakes""}, {""name"": ""Bailey""}]"' + in result_csv.output + ) + + def test_query_json_unicode_not_escaped_by_default(db_path): db = Database(db_path) with db.conn: diff --git a/tests/test_column_affinity.py b/tests/test_column_affinity.py index fa23345..aa2031d 100644 --- a/tests/test_column_affinity.py +++ b/tests/test_column_affinity.py @@ -32,6 +32,9 @@ EXAMPLES = [ ("BOOLEAN", float), ("DATE", float), ("DATETIME", float), + # Not a real SQLite affinity, but sqlite-utils treats it as TEXT so + # transform() does not mangle JSON columns - see issue 579 + ("JSON", str), ] diff --git a/tests/test_create.py b/tests/test_create.py index 40746bf..553eb1e 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -375,6 +375,12 @@ def test_create_error_if_invalid_self_referential_foreign_keys(fresh_db): 'CREATE TABLE "dogs" (\n "name" TEXT\n, "float" FLOAT)', ), ("blob", "blob", None, 'CREATE TABLE "dogs" (\n "name" TEXT\n, "blob" BLOB)'), + ( + "friends", + "JSON", + None, + 'CREATE TABLE "dogs" (\n "name" TEXT\n, "friends" JSON)', + ), ( "default_str", None,