Auto-decode columns declared as JSON in output modes (issue #579)

Columns declared with type JSON now decode automatically wherever nested
JSON is already supported (query/rows/search/memory output, and the
Python API), without needing --json-cols. Uses sqlite3's built-in
detect_types=PARSE_DECLTYPES + a registered JSON converter, so it works
transparently through joins and aliases. Flat formats (CSV/TSV/table/raw)
re-serialize decoded values back to JSON text instead of a Python repr.
Also adds "JSON" as a first-class column type for create()/add_column()/
transform(), and fixes column_affinity("JSON") so transform() doesn't
rewrite JSON columns as REAL.

sqlite_utils/cli.py

@@ -2323,14 +2323,14 @@ def _execute_query(
         cursor_or_rows: Any = cursor
         if raw:
             row = cursor_or_rows.fetchone()
-            data = row[0] if row else None
+            data = _flatten_json(row[0]) if row else None
             if isinstance(data, bytes):
                 sys.stdout.buffer.write(data)
             else:
                 sys.stdout.write(str(data))
         elif raw_lines:
             for row in cursor:
-                data = row[0]
+                data = _flatten_json(row[0])
                 if isinstance(data, bytes):
                     sys.stdout.buffer.write(data + b"\n")
                 else:
@@ -2341,7 +2341,7 @@ def _execute_query(
         elif fmt or table:
             print(
                 tabulate.tabulate(
-                    list(cursor),
+                    [[_flatten_json(value) for value in row] for row in cursor],
                     headers=() if no_headers else headers,
                     tablefmt=fmt or "simple",
                 )
@@ -2351,7 +2351,7 @@ def _execute_query(
             if not no_headers:
                 writer.writerow(headers)
             for row in cursor:
-                writer.writerow(row)
+                writer.writerow([_flatten_json(value) for value in row])
         else:
             for line in output_rows(cursor, headers, nl, arrays, json_cols, ascii_):
                 click.echo(line)
@@ -3826,7 +3826,9 @@ def output_rows(iterator, headers, nl, arrays, json_cols, ascii_=False):
 def output_transpose(rows, headers, no_headers):
     # psql-style extended display: one "key = value" block per row
     headers = [str(h) for h in headers]
-    str_rows = [["" if v is None else str(v) for v in row] for row in rows]
+    str_rows = [
+        ["" if v is None else str(_flatten_json(v)) for v in row] for row in rows
+    ]
     key_width = max([len(h) for h in headers], default=0)
     val_width = max([len(v) for row in str_rows for v in row], default=0)
     total_width = key_width + 3 + val_width
@@ -3837,6 +3839,15 @@ def output_transpose(rows, headers, no_headers):
             yield "{} | {}".format(header.ljust(key_width), value)
 
 
+def _flatten_json(value):
+    # Columns declared as JSON are auto-decoded to dict/list by the sqlite3
+    # driver (see issue 579) - flat output formats (CSV/TSV/table/raw) need
+    # that turned back into a JSON string rather than a Python repr
+    if isinstance(value, (dict, list)):
+        return json.dumps(value)
+    return value
+
+
 def maybe_json(value):
     if not isinstance(value, str):
         return value

sqlite_utils/db.py

@@ -335,6 +335,7 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = {
     "FLOAT": "FLOAT",
     "REAL": "REAL",
     "BLOB": "BLOB",
+    "JSON": "JSON",
     "text": "TEXT",
     "str": "TEXT",
     "integer": "INTEGER",
@@ -343,6 +344,7 @@ COLUMN_TYPE_MAPPING: dict[Any, str] = {
     "real": "REAL",
     "blob": "BLOB",
     "bytes": "BLOB",
+    "json": "JSON",
 }
 # If numpy is available, add more types
 if np:
@@ -532,11 +534,14 @@ class Database:
                 uri,
                 uri=True,
                 check_same_thread=False,
+                detect_types=sqlite3.PARSE_DECLTYPES,
             )
             self.memory = True
             self.memory_name = memory_name
         elif memory or filename_or_conn == ":memory:":
-            self.conn = sqlite3.connect(":memory:")
+            self.conn = sqlite3.connect(
+                ":memory:", detect_types=sqlite3.PARSE_DECLTYPES
+            )
             self.memory = True
         elif isinstance(filename_or_conn, (str, pathlib.Path)):
             if recreate and os.path.exists(filename_or_conn):
@@ -545,9 +550,13 @@ class Database:
                 except OSError:
                     # Avoid mypy and __repr__ errors, see:
                     # https://github.com/simonw/sqlite-utils/issues/503
-                    self.conn = sqlite3.connect(":memory:")
+                    self.conn = sqlite3.connect(
+                        ":memory:", detect_types=sqlite3.PARSE_DECLTYPES
+                    )
                     raise
-            self.conn = sqlite3.connect(str(filename_or_conn))
+            self.conn = sqlite3.connect(
+                str(filename_or_conn), detect_types=sqlite3.PARSE_DECLTYPES
+            )
         else:
             if recreate:
                 raise ValueError("recreate cannot be used with connections, only paths")

sqlite_utils/utils.py

@@ -170,6 +170,10 @@ def column_affinity(column_type: str) -> type:
     column_type = column_type.upper().strip()
     if column_type == "":
         return str  # We differ from spec, which says it should be BLOB
+    if "JSON" in column_type:
+        # Not a real SQLite affinity, but sqlite-utils treats JSON as a
+        # supported column type (see issue 579) and it is stored as text
+        return str
     if "INT" in column_type:
         return int
     if "CHAR" in column_type or "CLOB" in column_type or "TEXT" in column_type:
@@ -182,6 +186,19 @@ def column_affinity(column_type: str) -> type:
     return float
 
 
+def _json_column_converter(raw: bytes) -> Any:
+    # Registered as the sqlite3 converter for columns declared as JSON, see
+    # https://github.com/simonw/sqlite-utils/issues/579 - falls back to the
+    # decoded text unchanged if it does not turn out to be valid JSON
+    try:
+        return json.loads(raw)
+    except ValueError:
+        return raw.decode("utf-8", "replace")
+
+
+sqlite3.register_converter("JSON", _json_column_converter)
+
+
 def decode_base64_values(doc: dict[str, Any]) -> dict[str, Any]:
     # Looks for '{"$base64": true..., "encoded": ...}' values and decodes them
     to_fix = [