diff --git a/datasette/renderer.py b/datasette/renderer.py index 0e01f52f..ff6ffee7 100644 --- a/datasette/renderer.py +++ b/datasette/renderer.py @@ -6,26 +6,39 @@ from datasette.utils import ( error_body, path_from_row_pks, remove_infinites, - sqlite3, value_as_boolean, ) from datasette.utils.asgi import Response +def _load_json_value(value): + try: + return json.loads(value) + except (TypeError, ValueError): + return value + + def convert_specific_columns_to_json(rows, columns, json_cols): + if not rows: + return rows + if columns is None: + if not isinstance(rows[0], dict): + return rows + columns = rows[0].keys() json_cols = set(json_cols) if not json_cols.intersection(columns): return rows new_rows = [] for row in rows: - new_row = [] - for value, column in zip(row, columns): - if column in json_cols: - try: - value = json.loads(value) - except (TypeError, ValueError): - pass - new_row.append(value) + if isinstance(row, dict): + new_row = dict(row) + for column in json_cols.intersection(row): + new_row[column] = _load_json_value(row[column]) + else: + new_row = [ + _load_json_value(value) if column in json_cols else value + for value, column in zip(row, columns) + ] new_rows.append(new_row) return new_rows @@ -38,9 +51,9 @@ def json_renderer(request, args, data, error, truncated=None): json_cols = [] if "_json" in args: json_cols = args.getlist("_json") - if json_cols and "rows" in data and "columns" in data: + if json_cols and "rows" in data: data["rows"] = convert_specific_columns_to_json( - data["rows"], data["columns"], json_cols + data["rows"], data.get("columns"), json_cols ) # unless _json_infinity=1 requested, replace infinity with None @@ -61,11 +74,10 @@ def json_renderer(request, args, data, error, truncated=None): if shape == "arrayfirst": if not data["rows"]: data = [] - elif isinstance(data["rows"][0], sqlite3.Row): - data = [row[0] for row in data["rows"]] - else: - assert isinstance(data["rows"][0], dict) + elif isinstance(data["rows"][0], dict): data = [next(iter(row.values())) for row in data["rows"]] + else: + data = [row[0] for row in data["rows"]] elif shape in ("objects", "object", "array"): columns = data.get("columns") rows = data.get("rows") @@ -96,10 +108,10 @@ def json_renderer(request, args, data, error, truncated=None): elif shape == "arrays": if not data["rows"]: pass - elif isinstance(data["rows"][0], sqlite3.Row): - data["rows"] = [list(row) for row in data["rows"]] - else: + elif isinstance(data["rows"][0], dict): data["rows"] = [list(row.values()) for row in data["rows"]] + else: + data["rows"] = [list(row) for row in data["rows"]] else: status_code = 400 data = error_body(f"Invalid _shape: {shape}", status_code) diff --git a/tests/test_api.py b/tests/test_api.py index 690cb080..00e9d25e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -775,6 +775,95 @@ test_json_columns_default_expected = [ ] +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["table", "query"]) +@pytest.mark.parametrize("include_columns", [False, True]) +@pytest.mark.parametrize( + "shape", ["objects", "object", "array", "arrayfirst", "arrays"] +) +async def test_json_columns_with_output_shapes(endpoint, include_columns, shape): + if endpoint == "query" and shape == "object": + pytest.skip("Object shape requires a table primary key") + ds = Datasette() + db = ds.add_memory_database("json_shapes") + await db.execute_write("CREATE TABLE t (value text, id integer primary key)") + await db.execute_write("INSERT INTO t VALUES (?, ?)", ['["a", "b"]', 1]) + parameters = {"_json": "value", "_shape": shape} + if include_columns: + parameters["_extra"] = "columns" + if endpoint == "query": + parameters["sql"] = "SELECT value, id FROM t" + path = "/json_shapes/-/query.json" + else: + path = "/json_shapes/t.json" + try: + response = await ds.client.get(path + "?" + urllib.parse.urlencode(parameters)) + assert response.status_code == 200 + result = response.json() + row = {"value": ["a", "b"], "id": 1} + if shape == "object": + assert result == {"1": row} + elif shape == "array": + assert result == [row] + elif shape == "arrayfirst": + assert result == [["a", "b"]] + elif shape == "arrays": + assert result["rows"] == [[["a", "b"], 1]] + else: + assert result["rows"] == [row] + if shape in ("objects", "arrays"): + assert ("columns" in result) == include_columns + finally: + ds.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["table", "row"]) +@pytest.mark.parametrize( + "value,expected", [('["a"]', ["a"]), ("not JSON", "not JSON"), (None, None)] +) +async def test_json_columns_preserve_other_values(endpoint, value, expected): + ds = Datasette() + db = ds.add_memory_database("json_values") + await db.execute_write( + "CREATE TABLE t (id integer primary key, value text, other text)" + ) + await db.execute_write( + "INSERT INTO t VALUES (?, ?, ?)", [1, value, '{"keep": true}'] + ) + path = "/json_values/t.json" if endpoint == "table" else "/json_values/t/1.json" + try: + response = await ds.client.get(path + "?_json=value") + assert response.status_code == 200 + assert response.json()["rows"] == [ + {"id": 1, "value": expected, "other": '{"keep": true}'} + ] + finally: + ds.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "shape", ["objects", "object", "array", "arrayfirst", "arrays"] +) +async def test_json_columns_with_empty_results(shape): + ds = Datasette() + db = ds.add_memory_database("json_empty") + await db.execute_write("CREATE TABLE t (id integer primary key, value text)") + try: + response = await ds.client.get(f"/json_empty/t.json?_json=value&_shape={shape}") + assert response.status_code == 200 + result = response.json() + if shape == "object": + assert result == {} + elif shape in ("array", "arrayfirst"): + assert result == [] + else: + assert result["rows"] == [] + finally: + ds.close() + + @pytest.mark.asyncio @pytest.mark.parametrize( "extra_args,expected",