Auto-decode columns declared as JSON in output modes (issue #579)

Columns declared with type JSON now decode automatically wherever nested
JSON is already supported (query/rows/search/memory output, and the
Python API), without needing --json-cols. Uses sqlite3's built-in
detect_types=PARSE_DECLTYPES + a registered JSON converter, so it works
transparently through joins and aliases. Flat formats (CSV/TSV/table/raw)
re-serialize decoded values back to JSON text instead of a Python repr.
Also adds "JSON" as a first-class column type for create()/add_column()/
transform(), and fixes column_affinity("JSON") so transform() doesn't
rewrite JSON columns as REAL.

docs/cli.rst

@@ -192,7 +192,15 @@ If one of your columns contains JSON, by default it will be returned as an escap
         }
     ]
 
-You can use the ``--json-cols`` option to automatically detect these JSON columns and output them as nested JSON data:
+If the column is declared as type ``JSON`` in the table's schema, this happens automatically without needing ``--json-cols``:
+
+.. code-block:: bash
+
+    sqlite-utils dogs.db "CREATE TABLE dogs2 (id integer primary key, name text, friends JSON)"
+
+Columns declared this way are decoded as nested JSON by every command that returns JSON (``rows``, ``query``, ``search``, ``memory``), and by the Python library too. CSV, TSV and table output always show the underlying text, since flat formats cannot represent nested structures.
+
+For other TEXT columns that merely happen to contain a JSON string, you can use the ``--json-cols`` option to automatically detect these JSON columns and output them as nested JSON data:
 
 .. code-block:: bash
 

docs/python-api.rst

@@ -1564,12 +1564,13 @@ You can add a new column to a table using the ``.add_column(col_name, col_type)`
     db.table("dogs").add_column("dob", datetime.date)
     db.table("dogs").add_column("image", "BLOB")
     db.table("dogs").add_column("website") # str by default
+    db.table("dogs").add_column("friends", "JSON")
 
 You can specify the ``col_type`` argument either using a SQLite type as a string, or by directly passing a Python type e.g. ``str`` or ``float``.
 
 The ``col_type`` is optional - if you omit it the type of ``TEXT`` will be used.
 
-SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"`` or ``"BLOB"``.
+SQLite types you can specify are ``"TEXT"``, ``"INTEGER"``, ``"FLOAT"``, ``"REAL"``, ``"BLOB"`` or ``"JSON"``. A column declared as ``"JSON"`` will be automatically decoded back into a Python object (rather than a JSON string) whenever it is read, both by the Python library and by the :ref:`CLI <cli_json_values>`.
 
 If you pass a Python type, it will be mapped to SQLite types as shown here::
 

sqlite_utils/cli.py

@@ -2323,14 +2323,14 @@ def _execute_query(
         cursor_or_rows: Any = cursor
         if raw:
             row = cursor_or_rows.fetchone()
-            data = row[0] if row else None
+            data = _flatten_json(row[0]) if row else None
             if isinstance(data, bytes):
                 sys.stdout.buffer.write(data)
             else:
                 sys.stdout.write(str(data))
         elif raw_lines:
             for row in cursor:
-                data = row[0]
+                data = _flatten_json(row[0])
                 if isinstance(data, bytes):
                     sys.stdout.buffer.write(data + b"\n")
                 else:
@@ -2341,7 +2341,7 @@ def _execute_query(
         elif fmt or table:
             print(
                 tabulate.tabulate(
-                    list(cursor),
+                    [[_flatten_json(value) for value in row] for row in cursor],
                     headers=() if no_headers else headers,
                     tablefmt=fmt or "simple",
                 )
@@ -2351,7 +2351,7 @@ def _execute_query(
             if not no_headers:
                 writer.writerow(headers)
             for row in cursor:
-                writer.writerow(row)
+                writer.writerow([_flatten_json(value) for value in row])
         else:
             for line in output_rows(cursor, headers, nl, arrays, json_cols, ascii_):
                 click.echo(line)
@@ -3826,7 +3826,9 @@ def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False):
 def output_transpose(rows, headers, no_headers):
     # psql-style extended display: one "key = value" block per row
     headers = [str(h) for h in headers]
-    str_rows = [["" if v is None else str(v) for v in row] for row in rows]
+    str_rows = [
+        ["" if v is None else str(_flatten_json(v)) for v in row] for row in rows
+    ]
     key_width = max([len(h) for h in headers], default=0)
     val_width = max([len(v) for row in str_rows for v in row], default=0)
     total_width = key_width + 3 + val_width
@@ -3837,6 +3839,15 @@ def output_transpose(rows, headers, no_headers):
             yield "{} | {}".format(header.ljust(key_width), value)
 
 
+def _flatten_json(value):
+    # Columns declared as JSON are auto-decoded to dict/list by the sqlite3
+    # driver (see issue 579) - flat output formats (CSV/TSV/table/raw) need
+    # that turned back into a JSON string rather than a Python repr
+    if isinstance(value, (dict, list)):
+        return json.dumps(value)
+    return value
+
+
 def maybe_json(value):
     if not isinstance(value, str):
         return value

sqlite_utils/db.py

@@ -335,6 +335,7 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = {
     "FLOAT": "FLOAT",
     "REAL": "REAL",
     "BLOB": "BLOB",
+    "JSON": "JSON",
     "text": "TEXT",
     "str": "TEXT",
     "integer": "INTEGER",
@@ -343,6 +344,7 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = {
     "real": "REAL",
     "blob": "BLOB",
     "bytes": "BLOB",
+    "json": "JSON",
 }
 # If numpy is available, add more types
 if np:
@@ -532,11 +534,14 @@ class Database:
                 uri,
                 uri=True,
                 check_same_thread=False,
+                detect_types=sqlite3.PARSE_DECLTYPES,
             )
             self.memory = True
             self.memory_name = memory_name
         elif memory or filename_or_conn == ":memory:":
-            self.conn = sqlite3.connect(":memory:")
+            self.conn = sqlite3.connect(
+                ":memory:", detect_types=sqlite3.PARSE_DECLTYPES
+            )
             self.memory = True
         elif isinstance(filename_or_conn, (str, pathlib.Path)):
             if recreate and os.path.exists(filename_or_conn):
@@ -545,9 +550,13 @@ class Database:
                 except OSError:
                     # Avoid mypy and __repr__ errors, see:
                     # https://github.com/simonw/sqlite-utils/issues/503
-                    self.conn = sqlite3.connect(":memory:")
+                    self.conn = sqlite3.connect(
+                        ":memory:", detect_types=sqlite3.PARSE_DECLTYPES
+                    )
                     raise
-            self.conn = sqlite3.connect(str(filename_or_conn))
+            self.conn = sqlite3.connect(
+                str(filename_or_conn), detect_types=sqlite3.PARSE_DECLTYPES
+            )
         else:
             if recreate:
                 raise ValueError("recreate cannot be used with connections, only paths")

sqlite_utils/utils.py

@@ -170,6 +170,10 @@ def column_affinity(column_type: str) -> type:
     column_type = column_type.upper().strip()
     if column_type == "":
         return str  # We differ from spec, which says it should be BLOB
+    if "JSON" in column_type:
+        # Not a real SQLite affinity, but sqlite-utils treats JSON as a
+        # supported column type (see issue 579) and it is stored as text
+        return str
     if "INT" in column_type:
         return int
     if "CHAR" in column_type or "CLOB" in column_type or "TEXT" in column_type:
@@ -182,6 +186,19 @@ def column_affinity(column_type: str) -> type:
     return float
 
 
+def _json_column_converter(raw: bytes) -> Any:
+    # Registered as the sqlite3 converter for columns declared as JSON, see
+    # https://github.com/simonw/sqlite-utils/issues/579 - falls back to the
+    # decoded text unchanged if it does not turn out to be valid JSON
+    try:
+        return json.loads(raw)
+    except ValueError:
+        return raw.decode("utf-8", "replace")
+
+
+sqlite3.register_converter("JSON", _json_column_converter)
+
+
 def decode_base64_values(doc: dict[str, Any]) -> dict[str, Any]:
     # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them
     to_fix = [

tests/test_cli.py

@@ -1137,6 +1137,40 @@ def test_query_json_with_json_cols(db_path):
     assert expected == result_rows.output.strip()
 
 
+def test_query_json_declared_column_type_auto_decoded(db_path):
+    # Columns declared as JSON should be decoded automatically, without
+    # needing --json-cols - see https://github.com/simonw/sqlite-utils/issues/579
+    db = Database(db_path)
+    with db.conn:
+        db.execute(
+            "CREATE TABLE dogs (id INTEGER PRIMARY KEY, name TEXT, friends JSON)"
+        )
+        db["dogs"].insert(
+            {
+                "id": 1,
+                "name": "Cleo",
+                "friends": json.dumps([{"name": "Pancakes"}, {"name": "Bailey"}]),
+            }
+        )
+    expected = r"""
+    [{"id": 1, "name": "Cleo", "friends": [{"name": "Pancakes"}, {"name": "Bailey"}]}]
+    """.strip()
+    result = CliRunner().invoke(
+        cli.cli, [db_path, "select id, name, friends from dogs"]
+    )
+    assert expected == result.output.strip()
+    result_rows = CliRunner().invoke(cli.cli, ["rows", db_path, "dogs"])
+    assert expected == result_rows.output.strip()
+    # Flat formats should keep it as a JSON string, not a Python repr
+    result_csv = CliRunner().invoke(
+        cli.cli, [db_path, "select id, name, friends from dogs", "--csv"]
+    )
+    assert (
+        '1,Cleo,"[{""name"": ""Pancakes""}, {""name"": ""Bailey""}]"'
+        in result_csv.output
+    )
+
+
 def test_query_json_unicode_not_escaped_by_default(db_path):
     db = Database(db_path)
     with db.conn:

tests/test_column_affinity.py

@@ -32,6 +32,9 @@ EXAMPLES = [
     ("BOOLEAN", float),
     ("DATE", float),
     ("DATETIME", float),
+    # Not a real SQLite affinity, but sqlite-utils treats it as TEXT so
+    # transform() does not mangle JSON columns - see issue 579
+    ("JSON", str),
 ]
 
 

tests/test_create.py

@@ -375,6 +375,12 @@ def test_create_error_if_invalid_self_referential_foreign_keys(fresh_db):
             'CREATE TABLE "dogs" (\n   "name" TEXT\n, "float" FLOAT)',
         ),
         ("blob", "blob", None, 'CREATE TABLE "dogs" (\n   "name" TEXT\n, "blob" BLOB)'),
+        (
+            "friends",
+            "JSON",
+            None,
+            'CREATE TABLE "dogs" (\n   "name" TEXT\n, "friends" JSON)',
+        ),
         (
             "default_str",
             None,