From 3b24c88e9354cd0d78e0edfc2a3f7905b3e70ab1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 3 Jul 2026 14:05:45 -0700 Subject: [PATCH] Support editing BLOBs through JSON APIs --- datasette/utils/__init__.py | 30 ++++++ datasette/views/row.py | 9 +- datasette/views/table.py | 13 ++- datasette/views/table_create_alter.py | 6 ++ docs/binary_data.rst | 47 ++++++++++ docs/json_api.rst | 10 ++ tests/test_api_write.py | 127 ++++++++++++++++++++++++++ 7 files changed, 240 insertions(+), 2 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 1caff3a8..468828c5 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,4 +1,5 @@ import asyncio +import binascii from contextlib import contextmanager import aiofiles import click @@ -266,6 +267,35 @@ class CustomJSONEncoder(json.JSONEncoder): return json.JSONEncoder.default(self, obj) +class WriteJsonValueError(ValueError): + pass + + +def decode_write_json_cell(value): + if not isinstance(value, dict): + return value + keys = set(value.keys()) + if keys == {"$raw"}: + return value["$raw"] + if keys == {"$base64", "encoded"} and value.get("$base64") is True: + encoded = value["encoded"] + if not isinstance(encoded, str): + raise WriteJsonValueError("$base64 encoded value must be a string") + try: + return base64.b64decode(encoded, validate=True) + except binascii.Error as ex: + raise WriteJsonValueError("Invalid $base64 encoded value") from ex + return value + + +def decode_write_json_row(row): + return {key: decode_write_json_cell(value) for key, value in row.items()} + + +def decode_write_json_rows(rows): + return [decode_write_json_row(row) for row in rows] + + @contextmanager def sqlite_timelimit(conn, ms): deadline = time.perf_counter() + (ms / 1000) diff --git a/datasette/views/row.py b/datasette/views/row.py index 129216b9..f84d3a09 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -17,7 +17,9 @@ from datasette.utils import ( add_cors_headers, await_me_maybe, call_with_supported_arguments, + CustomJSONEncoder, CustomRow, + decode_write_json_row, InvalidSql, make_slot_function, path_from_row_pks, @@ -27,6 +29,7 @@ from datasette.utils import ( to_css_class, escape_sqlite, sqlite3, + WriteJsonValueError, ) from datasette.plugins import pm from datasette.extras import extra_names_from_request, ExtraScope @@ -806,6 +809,10 @@ class RowUpdateView(BaseView): return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) update = data["update"] + try: + update = decode_write_json_row(update) + except WriteJsonValueError as e: + return _error([str(e)], 400) # Validate column types from datasette.views.table import _validate_column_types @@ -867,4 +874,4 @@ class RowUpdateView(BaseView): self.ds.INFO, ) - return Response.json(result, status=200) + return Response.json(result, status=200, default=CustomJSONEncoder().default) diff --git a/datasette/views/table.py b/datasette/views/table.py index 1fc151e6..7899877d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -22,9 +22,11 @@ from datasette.utils import ( add_cors_headers, await_me_maybe, call_with_supported_arguments, + CustomJSONEncoder, CustomRow, append_querystring, compound_keys_after_sql, + decode_write_json_rows, format_bytes, make_slot_function, tilde_encode, @@ -41,6 +43,7 @@ from datasette.utils import ( urlsafe_components, value_as_boolean, InvalidSql, + WriteJsonValueError, sqlite3, ) from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response @@ -1057,6 +1060,10 @@ class TableInsertView(BaseView): ) if errors: return _error(errors, 400) + try: + rows = decode_write_json_rows(rows) + except WriteJsonValueError as e: + return _error([str(e)], 400) # Validate column types ct_errors = await _validate_column_types( @@ -1191,7 +1198,11 @@ class TableInsertView(BaseView): ) ) - return Response.json(result, status=200 if upsert else 201) + return Response.json( + result, + status=200 if upsert else 201, + default=CustomJSONEncoder().default, + ) class TableUpsertView(TableInsertView): diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index ffeb7f14..5feb1fa1 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -20,9 +20,11 @@ from datasette.column_types import SQLiteType from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( + decode_write_json_rows, escape_sqlite, get_outbound_foreign_keys, table_column_details, + WriteJsonValueError, ) from datasette.utils.asgi import NotFound, Response from datasette.utils.sqlite import sqlite_hidden_table_names @@ -838,6 +840,10 @@ class TableCreateView(BaseView): actor=request.actor, ): return _error(["Permission denied: need insert-row"], 403) + try: + rows = decode_write_json_rows(rows) + except WriteJsonValueError as e: + return _error([str(e)], 400) alter = False if rows: diff --git a/docs/binary_data.rst b/docs/binary_data.rst index 0c890fe5..fc972f98 100644 --- a/docs/binary_data.rst +++ b/docs/binary_data.rst @@ -12,6 +12,11 @@ Datasette includes special handling for these binary values. The Datasette inter :width: 311px :alt: Screenshot showing download links next to binary data in the table view +.. _binary_json_format: + +Binary values in JSON +--------------------- + Binary data is represented in ``.json`` exports using Base64 encoding. https://latest.datasette.io/fixtures/binary_data.json?_shape=array @@ -39,6 +44,48 @@ https://latest.datasette.io/fixtures/binary_data.json?_shape=array } ] +The same format can be used with the :ref:`JSON write API `. +If a column value in a ``row``, ``rows`` or ``update`` object is a JSON object with exactly ``"$base64"`` set to ``true`` and an ``"encoded"`` string, Datasette will decode that Base64 string and store the resulting bytes: + +.. code-block:: json + + { + "data": { + "$base64": true, + "encoded": "FRwCx60F/g==" + } + } + +This works for inserts, upserts and updates. It also works when creating a table from example ``row`` or ``rows`` data: Datasette decodes the value before inferring the schema, allowing that column to be created as a ``BLOB`` column. + +To store a JSON object with that exact shape literally, wrap it in a ``"$raw"`` object: + +.. code-block:: json + + { + "data": { + "$raw": { + "$base64": true, + "encoded": "FRwCx60F/g==" + } + } + } + +``"$raw"`` unwraps exactly one layer. To store a literal ``"$raw"`` object containing a Base64 object, wrap it again: + +.. code-block:: json + + { + "data": { + "$raw": { + "$raw": { + "$base64": true, + "encoded": "FRwCx60F/g==" + } + } + } + } + .. _binary_linking: Linking to binary downloads diff --git a/docs/json_api.rst b/docs/json_api.rst index eca22fdc..01b95f08 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -1532,6 +1532,8 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. +The row-based write APIs can write :ref:`binary values in JSON ` using Datasette's Base64 representation for BLOB data. + .. _ExecuteWriteView: Executing write SQL @@ -1660,6 +1662,8 @@ A single row can be inserted using the ``"row"`` key: } } +Column values can use the :ref:`binary value JSON format ` to write BLOB data. + If successful, this will return a ``201`` status code and the newly inserted row, for example: .. code-block:: json @@ -1765,6 +1769,8 @@ An upsert is an insert or update operation. If a row with a matching primary key The upsert API is mostly the same shape as the :ref:`insert API `. It requires both the :ref:`actions_insert_row` and :ref:`actions_update_row` permissions. +It also accepts the same :ref:`binary value JSON format `. + :: POST ///-/upsert @@ -1895,6 +1901,8 @@ To update a row, make a ``POST`` to ``//
//-/update``. You only need to pass the columns you want to update. Any other columns will be left unchanged. +Updated values can use the :ref:`binary value JSON format `. + If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. Add ``"return": true`` to the request body to return the updated row: @@ -2098,6 +2106,8 @@ Datasette will create a table with a schema that matches those rows and insert t "pk": "id" } +Example rows can use the :ref:`binary value JSON format `, allowing Datasette to infer ``BLOB`` columns. + Doing this requires both the :ref:`actions_create_table` and :ref:`actions_insert_row` permissions. The ``201`` response here will be similar to the ``columns`` form, but will also include the number of rows that were inserted as ``row_count``: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 76797742..1a4a966b 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -50,6 +50,133 @@ def _insert_and_fetch_created(conn, table, insert_sql): ).fetchone() +BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"} +BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}' + + +@pytest.mark.asyncio +async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "binary_create", + "row": { + "id": 1, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + "double_raw": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}}, + }, + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + assert "[data] BLOB" in response.json()["schema"] + assert "[literal] TEXT" in response.json()["schema"] + + rows = (await ds_write.get_database("data").execute(""" + select + typeof(data) as data_type, + hex(data) as data_hex, + typeof(literal) as literal_type, + literal, + typeof(double_raw) as double_raw_type, + double_raw + from binary_create + """)).dicts() + assert rows == [ + { + "data_type": "blob", + "data_hex": "000102FDFEFF", + "literal_type": "text", + "literal": BASE64_WRITE_API_LITERAL, + "double_raw_type": "text", + "double_raw": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}', + } + ] + + +@pytest.mark.asyncio +async def test_base64_write_api_insert_upsert_update_decode_blobs(ds_write): + token = write_token(ds_write) + db = ds_write.get_database("data") + await db.execute_write( + "create table binary_api (id integer primary key, data blob, literal text)" + ) + + insert_response = await ds_write.client.post( + "/data/binary_api/-/insert", + json={ + "row": { + "id": 1, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + } + }, + headers=_headers(token), + ) + assert insert_response.status_code == 201 + assert insert_response.json()["rows"][0]["data"] == BASE64_WRITE_API_VALUE + + upsert_response = await ds_write.client.post( + "/data/binary_api/-/upsert", + json={ + "rows": [ + { + "id": 2, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + } + ] + }, + headers=_headers(token), + ) + assert upsert_response.status_code == 200 + assert upsert_response.json() == {"ok": True} + + update_response = await ds_write.client.post( + "/data/binary_api/1/-/update", + json={ + "update": { + "data": {"$base64": True, "encoded": "/wAB"}, + "literal": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}}, + }, + "return": True, + }, + headers=_headers(token), + ) + assert update_response.status_code == 200 + assert update_response.json()["row"]["data"] == {"$base64": True, "encoded": "/wAB"} + + rows = (await db.execute(""" + select + id, + typeof(data) as data_type, + hex(data) as data_hex, + typeof(literal) as literal_type, + literal + from binary_api + order by id + """)).dicts() + assert rows == [ + { + "id": 1, + "data_type": "blob", + "data_hex": "FF0001", + "literal_type": "text", + "literal": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}', + }, + { + "id": 2, + "data_type": "blob", + "data_hex": "000102FDFEFF", + "literal_type": "text", + "literal": BASE64_WRITE_API_LITERAL, + }, + ] + + @pytest.mark.asyncio async def test_api_explorer_upsert_example_json(ds_write): response = await ds_write.client.get("/-/api", actor={"id": "root"})