Add opt-in Database(deserialize_json=True) to parse JSON strings back into dict/list on read

Values inserted as dict/list are serialized to JSON TEXT columns, but rows
fetched via .rows/.rows_where()/.get()/.search()/.query() previously always
came back as raw strings. This adds an opt-in constructor flag that parses
any string value that looks like a JSON object or array back into a
dict/list, without changing the default (raw string) behavior.

Fixes #612

docs/python-api.rst

@@ -2287,6 +2287,21 @@ For example:
     """).fetchall()
     # Returns [('Felton, CA',)]
 
+Values read back out through the Python API - via ``.rows``, ``.rows_where()``, ``.get()``,
+``.search()`` or ``.query()`` - come back as the raw JSON string that was stored, not as a
+``dict`` or ``list``, since that is the value SQLite actually returned. Pass
+``deserialize_json=True`` to the ``Database()`` constructor to opt in to automatically parsing
+any string value that looks like a JSON object or array back into a ``dict`` or ``list``:
+
+.. code-block:: python
+
+    db = sqlite_utils.Database("museums.db", deserialize_json=True)
+    print(db["niche_museums"].get(1))
+    # {'id': 1, 'name': 'The Bigfoot Discovery Museum', ..., 'hours': {'Monday': [11, 18], ...}}
+
+This defaults to ``False``, so existing code that expects raw strings continues to work
+unchanged.
+
 .. _python_api_conversions:
 
 Converting column values using SQL functions

sqlite_utils/db.py

@@ -501,6 +501,10 @@ class Database:
     :param use_old_upsert: set to ``True`` to force the older upsert implementation. See
       :ref:`python_api_old_upsert`
     :param strict: Apply STRICT mode to all created tables (unless overridden)
+    :param deserialize_json: set to ``True`` to automatically parse string column values that
+      look like JSON objects or arrays back into ``dict``/``list`` objects when rows are read
+      via ``.rows``, ``.rows_where()``, ``.get()``, ``.search()`` or ``.query()``. Defaults to
+      ``False``, so values are returned as the raw strings stored in the database.
     """
 
     _counts_table_name = "_counts"
@@ -519,10 +523,12 @@ class Database:
         execute_plugins: bool = True,
         use_old_upsert: bool = False,
         strict: bool = False,
+        deserialize_json: bool = False,
     ):
         self.memory_name = None
         self.memory = False
         self.use_old_upsert = use_old_upsert
+        self.deserialize_json = deserialize_json
         if not (
             (filename_or_conn is not None and (not memory and not memory_name))
             or (filename_or_conn is None and (memory or memory_name))
@@ -812,6 +818,13 @@ class Database:
         """.strip()
         self.execute(attach_sql)
 
+    def _row_dict(self, keys: Iterable[str], row: Sequence) -> dict:
+        "Build a row dict, deserializing JSON string values if self.deserialize_json is set."
+        d = dict(zip(keys, row))
+        if self.deserialize_json:
+            d = {key: dejsonify_if_needed(value) for key, value in d.items()}
+        return d
+
     def query(
         self, sql: str, params: Sequence | dict[str, Any] | None = None
     ) -> Generator[dict, None, None]:
@@ -857,7 +870,7 @@ class Database:
             if cursor.description is None:
                 raise ValueError(message)
             keys = dedupe_keys(d[0] for d in cursor.description)
-            return (dict(zip(keys, row)) for row in cursor)
+            return (self._row_dict(keys, row) for row in cursor)
         # Execute inside a savepoint, so a statement that turns out not to
         # return rows can be rolled back before the ValueError is raised
         self.conn.execute('SAVEPOINT "sqlite_utils_query"')
@@ -879,8 +892,8 @@ class Database:
                 fetched = cursor.fetchall()
                 self.conn.execute('RELEASE "sqlite_utils_query"')
                 released = True
-                return (dict(zip(keys, row)) for row in fetched)
-            return (dict(zip(keys, row)) for row in cursor)
+                return (self._row_dict(keys, row) for row in fetched)
+            return (self._row_dict(keys, row) for row in cursor)
         finally:
             if not released and self.conn.in_transaction:
                 # An error occurred - undo anything the statement changed.
@@ -2016,7 +2029,7 @@ class Queryable:
         cursor = self.db.execute(sql, where_args or [])
         columns = dedupe_keys(c[0] for c in cursor.description)
         for row in cursor:
-            yield dict(zip(columns, row))
+            yield self.db._row_dict(columns, row)
 
     def pks_and_rows_where(
         self,
@@ -3664,7 +3677,7 @@ class Table(Queryable):
         )
         columns = dedupe_keys(c[0] for c in cursor.description)
         for row in cursor:
-            yield dict(zip(columns, row))
+            yield self.db._row_dict(columns, row)
 
     def value_or_default(self, key: str, value: Any) -> Any:
         return self._defaults[key] if value is DEFAULT else value
@@ -5105,6 +5118,18 @@ def jsonify_if_needed(value: object) -> object:
         return value
 
 
+def dejsonify_if_needed(value: object) -> object:
+    "Parse a string that looks like a JSON object or array back into a dict/list."
+    if isinstance(value, str) and value[:1] in ("{", "["):
+        try:
+            parsed = json.loads(value)
+        except ValueError:
+            return value
+        if isinstance(parsed, (dict, list)):
+            return parsed
+    return value
+
+
 def resolve_extracts(
     extracts: dict[str, str] | list[str] | tuple[str] | None,
 ) -> dict:

tests/test_create.py

@@ -899,6 +899,28 @@ def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure):
     assert data_structure == json.loads(row[1])
 
 
+def test_deserialize_json(fresh_db):
+    fresh_db["test"].insert(
+        {"id": 1, "data": {"foo": "bar"}, "tags": [1, 2, 3], "name": "not json"},
+        pk="id",
+    )
+    # Default behaviour: values come back as the raw JSON strings that were stored
+    row = fresh_db["test"].get(1)
+    assert row["data"] == '{"foo": "bar"}'
+    assert row["tags"] == "[1, 2, 3]"
+    assert row["name"] == "not json"
+
+    # Opt-in: deserialize_json=True parses them back into dict/list
+    db2 = Database(fresh_db.conn, deserialize_json=True)
+    row2 = db2["test"].get(1)
+    assert row2["data"] == {"foo": "bar"}
+    assert row2["tags"] == [1, 2, 3]
+    assert row2["name"] == "not json"
+    # Also applies to .rows, .rows_where() and .query()
+    assert list(db2["test"].rows)[0]["data"] == {"foo": "bar"}
+    assert list(db2.query("select data from test"))[0]["data"] == {"foo": "bar"}
+
+
 def test_insert_list_nested_unicode(fresh_db):
     fresh_db["test"].insert(
         {"id": 1, "data": {"key1": {"nested": ["cømplex"]}}}, pk="id"