Decode JSON columns without changing table row values

Refs #3003

datasette/renderer.py

@@ -6,26 +6,39 @@ from datasette.utils import (
     error_body,
     path_from_row_pks,
     remove_infinites,
-    sqlite3,
     value_as_boolean,
 )
 from datasette.utils.asgi import Response
 
 
+def _load_json_value(value):
+    try:
+        return json.loads(value)
+    except (TypeError, ValueError):
+        return value
+
+
 def convert_specific_columns_to_json(rows, columns, json_cols):
+    if not rows:
+        return rows
+    if columns is None:
+        if not isinstance(rows[0], dict):
+            return rows
+        columns = rows[0].keys()
     json_cols = set(json_cols)
     if not json_cols.intersection(columns):
         return rows
     new_rows = []
     for row in rows:
-        new_row = []
-        for value, column in zip(row, columns):
-            if column in json_cols:
-                try:
-                    value = json.loads(value)
-                except (TypeError, ValueError):
-                    pass
-            new_row.append(value)
+        if isinstance(row, dict):
+            new_row = dict(row)
+            for column in json_cols.intersection(row):
+                new_row[column] = _load_json_value(row[column])
+        else:
+            new_row = [
+                _load_json_value(value) if column in json_cols else value
+                for value, column in zip(row, columns)
+            ]
         new_rows.append(new_row)
     return new_rows
 
@@ -38,9 +51,9 @@ def json_renderer(request, args, data, error, truncated=None):
     json_cols = []
     if "_json" in args:
         json_cols = args.getlist("_json")
-    if json_cols and "rows" in data and "columns" in data:
+    if json_cols and "rows" in data:
         data["rows"] = convert_specific_columns_to_json(
-            data["rows"], data["columns"], json_cols
+            data["rows"], data.get("columns"), json_cols
         )
 
     # unless _json_infinity=1 requested, replace infinity with None
@@ -61,11 +74,10 @@ def json_renderer(request, args, data, error, truncated=None):
     if shape == "arrayfirst":
         if not data["rows"]:
             data = []
-        elif isinstance(data["rows"][0], sqlite3.Row):
-            data = [row[0] for row in data["rows"]]
-        else:
-            assert isinstance(data["rows"][0], dict)
+        elif isinstance(data["rows"][0], dict):
             data = [next(iter(row.values())) for row in data["rows"]]
+        else:
+            data = [row[0] for row in data["rows"]]
     elif shape in ("objects", "object", "array"):
         columns = data.get("columns")
         rows = data.get("rows")
@@ -96,10 +108,10 @@ def json_renderer(request, args, data, error, truncated=None):
     elif shape == "arrays":
         if not data["rows"]:
             pass
-        elif isinstance(data["rows"][0], sqlite3.Row):
-            data["rows"] = [list(row) for row in data["rows"]]
-        else:
+        elif isinstance(data["rows"][0], dict):
             data["rows"] = [list(row.values()) for row in data["rows"]]
+        else:
+            data["rows"] = [list(row) for row in data["rows"]]
     else:
         status_code = 400
         data = error_body(f"Invalid _shape: {shape}", status_code)

tests/test_api.py

@@ -775,6 +775,95 @@ test_json_columns_default_expected = [
 ]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("endpoint", ["table", "query"])
+@pytest.mark.parametrize("include_columns", [False, True])
+@pytest.mark.parametrize(
+    "shape", ["objects", "object", "array", "arrayfirst", "arrays"]
+)
+async def test_json_columns_with_output_shapes(endpoint, include_columns, shape):
+    if endpoint == "query" and shape == "object":
+        pytest.skip("Object shape requires a table primary key")
+    ds = Datasette()
+    db = ds.add_memory_database("json_shapes")
+    await db.execute_write("CREATE TABLE t (value text, id integer primary key)")
+    await db.execute_write("INSERT INTO t VALUES (?, ?)", ['["a", "b"]', 1])
+    parameters = {"_json": "value", "_shape": shape}
+    if include_columns:
+        parameters["_extra"] = "columns"
+    if endpoint == "query":
+        parameters["sql"] = "SELECT value, id FROM t"
+        path = "/json_shapes/-/query.json"
+    else:
+        path = "/json_shapes/t.json"
+    try:
+        response = await ds.client.get(path + "?" + urllib.parse.urlencode(parameters))
+        assert response.status_code == 200
+        result = response.json()
+        row = {"value": ["a", "b"], "id": 1}
+        if shape == "object":
+            assert result == {"1": row}
+        elif shape == "array":
+            assert result == [row]
+        elif shape == "arrayfirst":
+            assert result == [["a", "b"]]
+        elif shape == "arrays":
+            assert result["rows"] == [[["a", "b"], 1]]
+        else:
+            assert result["rows"] == [row]
+        if shape in ("objects", "arrays"):
+            assert ("columns" in result) == include_columns
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("endpoint", ["table", "row"])
+@pytest.mark.parametrize(
+    "value,expected", [('["a"]', ["a"]), ("not JSON", "not JSON"), (None, None)]
+)
+async def test_json_columns_preserve_other_values(endpoint, value, expected):
+    ds = Datasette()
+    db = ds.add_memory_database("json_values")
+    await db.execute_write(
+        "CREATE TABLE t (id integer primary key, value text, other text)"
+    )
+    await db.execute_write(
+        "INSERT INTO t VALUES (?, ?, ?)", [1, value, '{"keep": true}']
+    )
+    path = "/json_values/t.json" if endpoint == "table" else "/json_values/t/1.json"
+    try:
+        response = await ds.client.get(path + "?_json=value")
+        assert response.status_code == 200
+        assert response.json()["rows"] == [
+            {"id": 1, "value": expected, "other": '{"keep": true}'}
+        ]
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "shape", ["objects", "object", "array", "arrayfirst", "arrays"]
+)
+async def test_json_columns_with_empty_results(shape):
+    ds = Datasette()
+    db = ds.add_memory_database("json_empty")
+    await db.execute_write("CREATE TABLE t (id integer primary key, value text)")
+    try:
+        response = await ds.client.get(f"/json_empty/t.json?_json=value&_shape={shape}")
+        assert response.status_code == 200
+        result = response.json()
+        if shape == "object":
+            assert result == {}
+        elif shape in ("array", "arrayfirst"):
+            assert result == []
+        else:
+            assert result["rows"] == []
+    finally:
+        ds.close()
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "extra_args,expected",