From 064625da59187592754512256939820e8a3b22e5 Mon Sep 17 00:00:00 2001 From: Parker Gurney Date: Wed, 5 Aug 2026 14:23:56 -0700 Subject: [PATCH] Add opt-in Database(deserialize_json=True) to parse JSON strings back into dict/list on read Values inserted as dict/list are serialized to JSON TEXT columns, but rows fetched via .rows/.rows_where()/.get()/.search()/.query() previously always came back as raw strings. This adds an opt-in constructor flag that parses any string value that looks like a JSON object or array back into a dict/list, without changing the default (raw string) behavior. Fixes #612 --- docs/python-api.rst | 15 +++++++++++++++ sqlite_utils/db.py | 35 ++++++++++++++++++++++++++++++----- tests/test_create.py | 22 ++++++++++++++++++++++ 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/docs/python-api.rst b/docs/python-api.rst index db7b995..beb9e00 100644 --- a/docs/python-api.rst +++ b/docs/python-api.rst @@ -2287,6 +2287,21 @@ For example: """).fetchall() # Returns [('Felton, CA',)] +Values read back out through the Python API - via ``.rows``, ``.rows_where()``, ``.get()``, +``.search()`` or ``.query()`` - come back as the raw JSON string that was stored, not as a +``dict`` or ``list``, since that is the value SQLite actually returned. Pass +``deserialize_json=True`` to the ``Database()`` constructor to opt in to automatically parsing +any string value that looks like a JSON object or array back into a ``dict`` or ``list``: + +.. code-block:: python + + db = sqlite_utils.Database("museums.db", deserialize_json=True) + print(db["niche_museums"].get(1)) + # {'id': 1, 'name': 'The Bigfoot Discovery Museum', ..., 'hours': {'Monday': [11, 18], ...}} + +This defaults to ``False``, so existing code that expects raw strings continues to work +unchanged. + .. _python_api_conversions: Converting column values using SQL functions diff --git a/sqlite_utils/db.py b/sqlite_utils/db.py index acc0356..2c08505 100644 --- a/sqlite_utils/db.py +++ b/sqlite_utils/db.py @@ -501,6 +501,10 @@ class Database: :param use_old_upsert: set to ``True`` to force the older upsert implementation. See :ref:`python_api_old_upsert` :param strict: Apply STRICT mode to all created tables (unless overridden) + :param deserialize_json: set to ``True`` to automatically parse string column values that + look like JSON objects or arrays back into ``dict``/``list`` objects when rows are read + via ``.rows``, ``.rows_where()``, ``.get()``, ``.search()`` or ``.query()``. Defaults to + ``False``, so values are returned as the raw strings stored in the database. """ _counts_table_name = "_counts" @@ -519,10 +523,12 @@ class Database: execute_plugins: bool = True, use_old_upsert: bool = False, strict: bool = False, + deserialize_json: bool = False, ): self.memory_name = None self.memory = False self.use_old_upsert = use_old_upsert + self.deserialize_json = deserialize_json if not ( (filename_or_conn is not None and (not memory and not memory_name)) or (filename_or_conn is None and (memory or memory_name)) @@ -812,6 +818,13 @@ class Database: """.strip() self.execute(attach_sql) + def _row_dict(self, keys: Iterable[str], row: Sequence) -> dict: + "Build a row dict, deserializing JSON string values if self.deserialize_json is set." + d = dict(zip(keys, row)) + if self.deserialize_json: + d = {key: dejsonify_if_needed(value) for key, value in d.items()} + return d + def query( self, sql: str, params: Sequence | dict[str, Any] | None = None ) -> Generator[dict, None, None]: @@ -857,7 +870,7 @@ class Database: if cursor.description is None: raise ValueError(message) keys = dedupe_keys(d[0] for d in cursor.description) - return (dict(zip(keys, row)) for row in cursor) + return (self._row_dict(keys, row) for row in cursor) # Execute inside a savepoint, so a statement that turns out not to # return rows can be rolled back before the ValueError is raised self.conn.execute('SAVEPOINT "sqlite_utils_query"') @@ -879,8 +892,8 @@ class Database: fetched = cursor.fetchall() self.conn.execute('RELEASE "sqlite_utils_query"') released = True - return (dict(zip(keys, row)) for row in fetched) - return (dict(zip(keys, row)) for row in cursor) + return (self._row_dict(keys, row) for row in fetched) + return (self._row_dict(keys, row) for row in cursor) finally: if not released and self.conn.in_transaction: # An error occurred - undo anything the statement changed. @@ -2016,7 +2029,7 @@ class Queryable: cursor = self.db.execute(sql, where_args or []) columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: - yield dict(zip(columns, row)) + yield self.db._row_dict(columns, row) def pks_and_rows_where( self, @@ -3664,7 +3677,7 @@ class Table(Queryable): ) columns = dedupe_keys(c[0] for c in cursor.description) for row in cursor: - yield dict(zip(columns, row)) + yield self.db._row_dict(columns, row) def value_or_default(self, key: str, value: Any) -> Any: return self._defaults[key] if value is DEFAULT else value @@ -5105,6 +5118,18 @@ def jsonify_if_needed(value: object) -> object: return value +def dejsonify_if_needed(value: object) -> object: + "Parse a string that looks like a JSON object or array back into a dict/list." + if isinstance(value, str) and value[:1] in ("{", "["): + try: + parsed = json.loads(value) + except ValueError: + return value + if isinstance(parsed, (dict, list)): + return parsed + return value + + def resolve_extracts( extracts: dict[str, str] | list[str] | tuple[str] | None, ) -> dict: diff --git a/tests/test_create.py b/tests/test_create.py index 553eb1e..4ad56cd 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -899,6 +899,28 @@ def test_insert_dictionaries_and_lists_as_json(fresh_db, data_structure): assert data_structure == json.loads(row[1]) +def test_deserialize_json(fresh_db): + fresh_db["test"].insert( + {"id": 1, "data": {"foo": "bar"}, "tags": [1, 2, 3], "name": "not json"}, + pk="id", + ) + # Default behaviour: values come back as the raw JSON strings that were stored + row = fresh_db["test"].get(1) + assert row["data"] == '{"foo": "bar"}' + assert row["tags"] == "[1, 2, 3]" + assert row["name"] == "not json" + + # Opt-in: deserialize_json=True parses them back into dict/list + db2 = Database(fresh_db.conn, deserialize_json=True) + row2 = db2["test"].get(1) + assert row2["data"] == {"foo": "bar"} + assert row2["tags"] == [1, 2, 3] + assert row2["name"] == "not json" + # Also applies to .rows, .rows_where() and .query() + assert list(db2["test"].rows)[0]["data"] == {"foo": "bar"} + assert list(db2.query("select data from test"))[0]["data"] == {"foo": "bar"} + + def test_insert_list_nested_unicode(fresh_db): fresh_db["test"].insert( {"id": 1, "data": {"key1": {"nested": ["cømplex"]}}}, pk="id"