From 3b24c88e9354cd0d78e0edfc2a3f7905b3e70ab1 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 3 Jul 2026 14:05:45 -0700 Subject: [PATCH 1/7] Support editing BLOBs through JSON APIs --- datasette/utils/__init__.py | 30 ++++++ datasette/views/row.py | 9 +- datasette/views/table.py | 13 ++- datasette/views/table_create_alter.py | 6 ++ docs/binary_data.rst | 47 ++++++++++ docs/json_api.rst | 10 ++ tests/test_api_write.py | 127 ++++++++++++++++++++++++++ 7 files changed, 240 insertions(+), 2 deletions(-) diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 1caff3a8..468828c5 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -1,4 +1,5 @@ import asyncio +import binascii from contextlib import contextmanager import aiofiles import click @@ -266,6 +267,35 @@ class CustomJSONEncoder(json.JSONEncoder): return json.JSONEncoder.default(self, obj) +class WriteJsonValueError(ValueError): + pass + + +def decode_write_json_cell(value): + if not isinstance(value, dict): + return value + keys = set(value.keys()) + if keys == {"$raw"}: + return value["$raw"] + if keys == {"$base64", "encoded"} and value.get("$base64") is True: + encoded = value["encoded"] + if not isinstance(encoded, str): + raise WriteJsonValueError("$base64 encoded value must be a string") + try: + return base64.b64decode(encoded, validate=True) + except binascii.Error as ex: + raise WriteJsonValueError("Invalid $base64 encoded value") from ex + return value + + +def decode_write_json_row(row): + return {key: decode_write_json_cell(value) for key, value in row.items()} + + +def decode_write_json_rows(rows): + return [decode_write_json_row(row) for row in rows] + + @contextmanager def sqlite_timelimit(conn, ms): deadline = time.perf_counter() + (ms / 1000) diff --git a/datasette/views/row.py b/datasette/views/row.py index 129216b9..f84d3a09 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -17,7 +17,9 @@ from datasette.utils import ( add_cors_headers, await_me_maybe, call_with_supported_arguments, + CustomJSONEncoder, CustomRow, + decode_write_json_row, InvalidSql, make_slot_function, path_from_row_pks, @@ -27,6 +29,7 @@ from datasette.utils import ( to_css_class, escape_sqlite, sqlite3, + WriteJsonValueError, ) from datasette.plugins import pm from datasette.extras import extra_names_from_request, ExtraScope @@ -806,6 +809,10 @@ class RowUpdateView(BaseView): return _error(["Invalid keys: {}".format(", ".join(invalid_keys))]) update = data["update"] + try: + update = decode_write_json_row(update) + except WriteJsonValueError as e: + return _error([str(e)], 400) # Validate column types from datasette.views.table import _validate_column_types @@ -867,4 +874,4 @@ class RowUpdateView(BaseView): self.ds.INFO, ) - return Response.json(result, status=200) + return Response.json(result, status=200, default=CustomJSONEncoder().default) diff --git a/datasette/views/table.py b/datasette/views/table.py index 1fc151e6..7899877d 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -22,9 +22,11 @@ from datasette.utils import ( add_cors_headers, await_me_maybe, call_with_supported_arguments, + CustomJSONEncoder, CustomRow, append_querystring, compound_keys_after_sql, + decode_write_json_rows, format_bytes, make_slot_function, tilde_encode, @@ -41,6 +43,7 @@ from datasette.utils import ( urlsafe_components, value_as_boolean, InvalidSql, + WriteJsonValueError, sqlite3, ) from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response @@ -1057,6 +1060,10 @@ class TableInsertView(BaseView): ) if errors: return _error(errors, 400) + try: + rows = decode_write_json_rows(rows) + except WriteJsonValueError as e: + return _error([str(e)], 400) # Validate column types ct_errors = await _validate_column_types( @@ -1191,7 +1198,11 @@ class TableInsertView(BaseView): ) ) - return Response.json(result, status=200 if upsert else 201) + return Response.json( + result, + status=200 if upsert else 201, + default=CustomJSONEncoder().default, + ) class TableUpsertView(TableInsertView): diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index ffeb7f14..5feb1fa1 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -20,9 +20,11 @@ from datasette.column_types import SQLiteType from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent from datasette.resources import DatabaseResource, TableResource from datasette.utils import ( + decode_write_json_rows, escape_sqlite, get_outbound_foreign_keys, table_column_details, + WriteJsonValueError, ) from datasette.utils.asgi import NotFound, Response from datasette.utils.sqlite import sqlite_hidden_table_names @@ -838,6 +840,10 @@ class TableCreateView(BaseView): actor=request.actor, ): return _error(["Permission denied: need insert-row"], 403) + try: + rows = decode_write_json_rows(rows) + except WriteJsonValueError as e: + return _error([str(e)], 400) alter = False if rows: diff --git a/docs/binary_data.rst b/docs/binary_data.rst index 0c890fe5..fc972f98 100644 --- a/docs/binary_data.rst +++ b/docs/binary_data.rst @@ -12,6 +12,11 @@ Datasette includes special handling for these binary values. The Datasette inter :width: 311px :alt: Screenshot showing download links next to binary data in the table view +.. _binary_json_format: + +Binary values in JSON +--------------------- + Binary data is represented in ``.json`` exports using Base64 encoding. https://latest.datasette.io/fixtures/binary_data.json?_shape=array @@ -39,6 +44,48 @@ https://latest.datasette.io/fixtures/binary_data.json?_shape=array } ] +The same format can be used with the :ref:`JSON write API `. +If a column value in a ``row``, ``rows`` or ``update`` object is a JSON object with exactly ``"$base64"`` set to ``true`` and an ``"encoded"`` string, Datasette will decode that Base64 string and store the resulting bytes: + +.. code-block:: json + + { + "data": { + "$base64": true, + "encoded": "FRwCx60F/g==" + } + } + +This works for inserts, upserts and updates. It also works when creating a table from example ``row`` or ``rows`` data: Datasette decodes the value before inferring the schema, allowing that column to be created as a ``BLOB`` column. + +To store a JSON object with that exact shape literally, wrap it in a ``"$raw"`` object: + +.. code-block:: json + + { + "data": { + "$raw": { + "$base64": true, + "encoded": "FRwCx60F/g==" + } + } + } + +``"$raw"`` unwraps exactly one layer. To store a literal ``"$raw"`` object containing a Base64 object, wrap it again: + +.. code-block:: json + + { + "data": { + "$raw": { + "$raw": { + "$base64": true, + "encoded": "FRwCx60F/g==" + } + } + } + } + .. _binary_linking: Linking to binary downloads diff --git a/docs/json_api.rst b/docs/json_api.rst index eca22fdc..01b95f08 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -1532,6 +1532,8 @@ The JSON write API Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`. +The row-based write APIs can write :ref:`binary values in JSON ` using Datasette's Base64 representation for BLOB data. + .. _ExecuteWriteView: Executing write SQL @@ -1660,6 +1662,8 @@ A single row can be inserted using the ``"row"`` key: } } +Column values can use the :ref:`binary value JSON format ` to write BLOB data. + If successful, this will return a ``201`` status code and the newly inserted row, for example: .. code-block:: json @@ -1765,6 +1769,8 @@ An upsert is an insert or update operation. If a row with a matching primary key The upsert API is mostly the same shape as the :ref:`insert API `. It requires both the :ref:`actions_insert_row` and :ref:`actions_update_row` permissions. +It also accepts the same :ref:`binary value JSON format `. + :: POST ///-/upsert @@ -1895,6 +1901,8 @@ To update a row, make a ``POST`` to ``//
//-/update``. You only need to pass the columns you want to update. Any other columns will be left unchanged. +Updated values can use the :ref:`binary value JSON format `. + If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body. Add ``"return": true`` to the request body to return the updated row: @@ -2098,6 +2106,8 @@ Datasette will create a table with a schema that matches those rows and insert t "pk": "id" } +Example rows can use the :ref:`binary value JSON format `, allowing Datasette to infer ``BLOB`` columns. + Doing this requires both the :ref:`actions_create_table` and :ref:`actions_insert_row` permissions. The ``201`` response here will be similar to the ``columns`` form, but will also include the number of rows that were inserted as ``row_count``: diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 76797742..1a4a966b 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -50,6 +50,133 @@ def _insert_and_fetch_created(conn, table, insert_sql): ).fetchone() +BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"} +BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}' + + +@pytest.mark.asyncio +async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write): + token = write_token(ds_write) + response = await ds_write.client.post( + "/data/-/create", + json={ + "table": "binary_create", + "row": { + "id": 1, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + "double_raw": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}}, + }, + "pk": "id", + }, + headers=_headers(token), + ) + assert response.status_code == 201 + assert "[data] BLOB" in response.json()["schema"] + assert "[literal] TEXT" in response.json()["schema"] + + rows = (await ds_write.get_database("data").execute(""" + select + typeof(data) as data_type, + hex(data) as data_hex, + typeof(literal) as literal_type, + literal, + typeof(double_raw) as double_raw_type, + double_raw + from binary_create + """)).dicts() + assert rows == [ + { + "data_type": "blob", + "data_hex": "000102FDFEFF", + "literal_type": "text", + "literal": BASE64_WRITE_API_LITERAL, + "double_raw_type": "text", + "double_raw": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}', + } + ] + + +@pytest.mark.asyncio +async def test_base64_write_api_insert_upsert_update_decode_blobs(ds_write): + token = write_token(ds_write) + db = ds_write.get_database("data") + await db.execute_write( + "create table binary_api (id integer primary key, data blob, literal text)" + ) + + insert_response = await ds_write.client.post( + "/data/binary_api/-/insert", + json={ + "row": { + "id": 1, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + } + }, + headers=_headers(token), + ) + assert insert_response.status_code == 201 + assert insert_response.json()["rows"][0]["data"] == BASE64_WRITE_API_VALUE + + upsert_response = await ds_write.client.post( + "/data/binary_api/-/upsert", + json={ + "rows": [ + { + "id": 2, + "data": BASE64_WRITE_API_VALUE, + "literal": {"$raw": BASE64_WRITE_API_VALUE}, + } + ] + }, + headers=_headers(token), + ) + assert upsert_response.status_code == 200 + assert upsert_response.json() == {"ok": True} + + update_response = await ds_write.client.post( + "/data/binary_api/1/-/update", + json={ + "update": { + "data": {"$base64": True, "encoded": "/wAB"}, + "literal": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}}, + }, + "return": True, + }, + headers=_headers(token), + ) + assert update_response.status_code == 200 + assert update_response.json()["row"]["data"] == {"$base64": True, "encoded": "/wAB"} + + rows = (await db.execute(""" + select + id, + typeof(data) as data_type, + hex(data) as data_hex, + typeof(literal) as literal_type, + literal + from binary_api + order by id + """)).dicts() + assert rows == [ + { + "id": 1, + "data_type": "blob", + "data_hex": "FF0001", + "literal_type": "text", + "literal": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}', + }, + { + "id": 2, + "data_type": "blob", + "data_hex": "000102FDFEFF", + "literal_type": "text", + "literal": BASE64_WRITE_API_LITERAL, + }, + ] + + @pytest.mark.asyncio async def test_api_explorer_upsert_example_json(ds_write): response = await ds_write.client.get("/-/api", actor={"id": "root"}) From 8856914be805f4dee2e241895b848031bae7c526 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 3 Jul 2026 14:45:38 -0700 Subject: [PATCH 2/7] Add column_details JSON extra --- datasette/views/table_extras.py | 37 +++++++++++++++++++++++++++++ docs/changelog.rst | 7 ++++++ docs/json_api.rst | 36 +++++++++++++++++++++++++++++ tests/test_api.py | 36 ++++++++++++++++++++++++++++- tests/test_table_api.py | 41 +++++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 1 deletion(-) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index db659c80..5ba4187a 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -1,6 +1,7 @@ import itertools from dataclasses import dataclass +from datasette.column_types import SQLiteType from datasette.database import QueryInterrupted from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider from datasette.plugins import pm @@ -342,6 +343,41 @@ class PrimaryKeysExtra(Extra): return context.pks +def column_detail_as_json(column): + return { + "type": column.type, + "sqlite_type": SQLiteType.from_declared_type(column.type).value, + "notnull": column.notnull, + "default": column.default_value, + "is_pk": bool(column.is_pk), + "hidden": bool(column.hidden), + } + + +class ColumnDetailsExtra(Extra): + description = ( + "SQLite schema details for columns in this table. The dictionary maps " + "column names to ``type``, ``sqlite_type``, ``notnull``, ``default``, " + "``is_pk`` and ``hidden`` values." + ) + example = ExtraExample("/fixtures/binary_data.json?_size=0&_extra=column_details") + examples = { + ExtraScope.ROW: ExtraExample( + "/fixtures/binary_data/1.json?_extra=column_details" + ) + } + scopes = {ExtraScope.TABLE, ExtraScope.ROW} + + async def resolve(self, context): + column_details = await context.datasette._get_resource_column_details( + context.database_name, context.table_name + ) + return { + column_name: column_detail_as_json(column) + for column_name, column in column_details.items() + } + + class ActionsExtra(Extra): description = 'Async callable returning table or view actions made available by core and plugin hooks. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions`, :ref:`plugin_hook_table_actions` and :ref:`plugin_hook_view_actions`.' scopes = {ExtraScope.TABLE} @@ -1206,6 +1242,7 @@ TABLE_EXTRA_CLASSES = [ ColumnsExtra, AllColumnsExtra, PrimaryKeysExtra, + ColumnDetailsExtra, DisplayColumnsAndRowsProvider, DisplayColumnsExtra, DisplayRowsExtra, diff --git a/docs/changelog.rst b/docs/changelog.rst index ec32b57e..7f3fd2c4 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -4,6 +4,13 @@ Changelog ========= +.. _v1_0_a36: + +1.0a36 (in development) +----------------------- + +- The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata. + .. _v1_0_a35: 1.0a35 (2026-06-23) diff --git a/docs/json_api.rst b/docs/json_api.rst index 01b95f08..05c2565f 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -402,6 +402,24 @@ The available table extras are listed below. "pk" ] +``column_details`` + SQLite schema details for columns in this table. The dictionary maps column names to ``type``, ``sqlite_type``, ``notnull``, ``default``, ``is_pk`` and ``hidden`` values. + + ``GET /fixtures/binary_data.json?_size=0&_extra=column_details`` + + .. code-block:: json + + { + "data": { + "type": "BLOB", + "sqlite_type": "BLOB", + "notnull": 0, + "default": null, + "is_pk": false, + "hidden": false + } + } + ``display_columns`` Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys. @@ -807,6 +825,24 @@ The following extras are available for row JSON responses. "id" ] +``column_details`` + SQLite schema details for columns in this table. The dictionary maps column names to ``type``, ``sqlite_type``, ``notnull``, ``default``, ``is_pk`` and ``hidden`` values. + + ``GET /fixtures/binary_data/1.json?_extra=column_details`` + + .. code-block:: json + + { + "data": { + "type": "BLOB", + "sqlite_type": "BLOB", + "notnull": 0, + "default": null, + "is_pk": false, + "hidden": false + } + } + ``render_cell`` Rendered HTML for each cell using the render_cell plugin hook (See the :ref:`render_cell() plugin hook ` documentation.) diff --git a/tests/test_api.py b/tests/test_api.py index f57d0206..da9e5d00 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -429,7 +429,7 @@ async def test_row_foreign_key_tables(ds_client): @pytest.mark.asyncio async def test_row_extras(ds_client): response = await ds_client.get( - "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables" + "/fixtures/simple_primary_key/1.json?_extra=database,table,primary_keys,query,request,debug,foreign_key_tables,column_details" ) assert response.status_code == 200 data = response.json() @@ -446,6 +446,40 @@ async def test_row_extras(ds_client): "format": "json", } assert len(data["foreign_key_tables"]) == 5 + assert data["column_details"]["id"] == { + "type": "INTEGER", + "sqlite_type": "INTEGER", + "notnull": 0, + "default": None, + "is_pk": True, + "hidden": False, + } + assert data["column_details"]["content"] == { + "type": "TEXT", + "sqlite_type": "TEXT", + "notnull": 0, + "default": None, + "is_pk": False, + "hidden": False, + } + + +@pytest.mark.asyncio +async def test_column_details_extra_row_for_null_blob(ds_client): + response = await ds_client.get( + "/fixtures/binary_data/3.json?_extra=column_details" + ) + assert response.status_code == 200 + assert response.json()["column_details"] == { + "data": { + "type": "BLOB", + "sqlite_type": "BLOB", + "notnull": 0, + "default": None, + "is_pk": False, + "hidden": False, + } + } @pytest.mark.asyncio diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 272e39e3..0362c701 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1336,6 +1336,47 @@ async def test_binary_data_in_json(ds_client, path, expected_json, expected_text assert response.text == expected_text +@pytest.mark.asyncio +async def test_column_details_extra_table(ds_client): + response = await ds_client.get( + "/fixtures/binary_data.json?_size=0&_extra=column_details" + ) + assert response.status_code == 200 + assert response.json()["column_details"] == { + "data": { + "type": "BLOB", + "sqlite_type": "BLOB", + "notnull": 0, + "default": None, + "is_pk": False, + "hidden": False, + } + } + + response = await ds_client.get( + "/fixtures/simple_primary_key.json?_size=0&_extra=column_details" + ) + assert response.status_code == 200 + assert response.json()["column_details"] == { + "id": { + "type": "INTEGER", + "sqlite_type": "INTEGER", + "notnull": 0, + "default": None, + "is_pk": True, + "hidden": False, + }, + "content": { + "type": "TEXT", + "sqlite_type": "TEXT", + "notnull": 0, + "default": None, + "is_pk": False, + "hidden": False, + }, + } + + @pytest.mark.asyncio @pytest.mark.parametrize( "qs", From b476218edb78447529d3085b3c1684a39aa80d62 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 3 Jul 2026 16:08:34 -0700 Subject: [PATCH 3/7] Refine column_details metadata shape --- datasette/views/table_extras.py | 22 ++++++++-- docs/json_api.rst | 14 ++++--- tests/test_api.py | 19 ++++----- tests/test_table_api.py | 71 ++++++++++++++++++++++++++++++--- 4 files changed, 101 insertions(+), 25 deletions(-) diff --git a/datasette/views/table_extras.py b/datasette/views/table_extras.py index 5ba4187a..274c99ff 100644 --- a/datasette/views/table_extras.py +++ b/datasette/views/table_extras.py @@ -347,18 +347,32 @@ def column_detail_as_json(column): return { "type": column.type, "sqlite_type": SQLiteType.from_declared_type(column.type).value, - "notnull": column.notnull, + "notnull": bool(column.notnull), "default": column.default_value, "is_pk": bool(column.is_pk), - "hidden": bool(column.hidden), + "pk_position": column.is_pk, + "hidden": column.hidden, } class ColumnDetailsExtra(Extra): description = ( "SQLite schema details for columns in this table. The dictionary maps " - "column names to ``type``, ``sqlite_type``, ``notnull``, ``default``, " - "``is_pk`` and ``hidden`` values." + "column names to objects describing the schema for each column." + ) + docs_note = ( + "Each object has ``type`` as the declared type string returned by " + 'SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the ' + "normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, " + "``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` " + 'as the raw SQL default expression string, such as ``"42"``, ' + "``\"'hello'\"`` or ``\"datetime('now')\"``, or ``null`` if there is " + "no default; ``is_pk`` as a boolean; ``pk_position`` as the integer " + "primary key position reported by SQLite, or ``0`` for columns that " + "are not part of the primary key; and ``hidden`` as the integer value " + "reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for " + "normal columns, ``1`` for hidden virtual table columns, ``2`` for " + "virtual generated columns and ``3`` for stored generated columns." ) example = ExtraExample("/fixtures/binary_data.json?_size=0&_extra=column_details") examples = { diff --git a/docs/json_api.rst b/docs/json_api.rst index 05c2565f..f14a5c07 100644 --- a/docs/json_api.rst +++ b/docs/json_api.rst @@ -403,7 +403,7 @@ The available table extras are listed below. ] ``column_details`` - SQLite schema details for columns in this table. The dictionary maps column names to ``type``, ``sqlite_type``, ``notnull``, ``default``, ``is_pk`` and ``hidden`` values. + SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.) ``GET /fixtures/binary_data.json?_size=0&_extra=column_details`` @@ -413,10 +413,11 @@ The available table extras are listed below. "data": { "type": "BLOB", "sqlite_type": "BLOB", - "notnull": 0, + "notnull": false, "default": null, "is_pk": false, - "hidden": false + "pk_position": 0, + "hidden": 0 } } @@ -826,7 +827,7 @@ The following extras are available for row JSON responses. ] ``column_details`` - SQLite schema details for columns in this table. The dictionary maps column names to ``type``, ``sqlite_type``, ``notnull``, ``default``, ``is_pk`` and ``hidden`` values. + SQLite schema details for columns in this table. The dictionary maps column names to objects describing the schema for each column. (Each object has ``type`` as the declared type string returned by SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, ``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` as the raw SQL default expression string, such as ``"42"``, ``"'hello'"`` or ``"datetime('now')"``, or ``null`` if there is no default; ``is_pk`` as a boolean; ``pk_position`` as the integer primary key position reported by SQLite, or ``0`` for columns that are not part of the primary key; and ``hidden`` as the integer value reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for normal columns, ``1`` for hidden virtual table columns, ``2`` for virtual generated columns and ``3`` for stored generated columns.) ``GET /fixtures/binary_data/1.json?_extra=column_details`` @@ -836,10 +837,11 @@ The following extras are available for row JSON responses. "data": { "type": "BLOB", "sqlite_type": "BLOB", - "notnull": 0, + "notnull": false, "default": null, "is_pk": false, - "hidden": false + "pk_position": 0, + "hidden": 0 } } diff --git a/tests/test_api.py b/tests/test_api.py index da9e5d00..ee3329e1 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -449,35 +449,36 @@ async def test_row_extras(ds_client): assert data["column_details"]["id"] == { "type": "INTEGER", "sqlite_type": "INTEGER", - "notnull": 0, + "notnull": False, "default": None, "is_pk": True, - "hidden": False, + "pk_position": 1, + "hidden": 0, } assert data["column_details"]["content"] == { "type": "TEXT", "sqlite_type": "TEXT", - "notnull": 0, + "notnull": False, "default": None, "is_pk": False, - "hidden": False, + "pk_position": 0, + "hidden": 0, } @pytest.mark.asyncio async def test_column_details_extra_row_for_null_blob(ds_client): - response = await ds_client.get( - "/fixtures/binary_data/3.json?_extra=column_details" - ) + response = await ds_client.get("/fixtures/binary_data/3.json?_extra=column_details") assert response.status_code == 200 assert response.json()["column_details"] == { "data": { "type": "BLOB", "sqlite_type": "BLOB", - "notnull": 0, + "notnull": False, "default": None, "is_pk": False, - "hidden": False, + "pk_position": 0, + "hidden": 0, } } diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 0362c701..067a8bab 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1346,10 +1346,11 @@ async def test_column_details_extra_table(ds_client): "data": { "type": "BLOB", "sqlite_type": "BLOB", - "notnull": 0, + "notnull": False, "default": None, "is_pk": False, - "hidden": False, + "pk_position": 0, + "hidden": 0, } } @@ -1361,21 +1362,79 @@ async def test_column_details_extra_table(ds_client): "id": { "type": "INTEGER", "sqlite_type": "INTEGER", - "notnull": 0, + "notnull": False, "default": None, "is_pk": True, - "hidden": False, + "pk_position": 1, + "hidden": 0, }, "content": { "type": "TEXT", "sqlite_type": "TEXT", - "notnull": 0, + "notnull": False, "default": None, "is_pk": False, - "hidden": False, + "pk_position": 0, + "hidden": 0, }, } + response = await ds_client.get( + "/fixtures/compound_three_primary_keys.json?_size=0&_extra=column_details" + ) + assert response.status_code == 200 + column_details = response.json()["column_details"] + assert column_details["pk1"]["is_pk"] is True + assert column_details["pk1"]["pk_position"] == 1 + assert column_details["pk2"]["is_pk"] is True + assert column_details["pk2"]["pk_position"] == 2 + assert column_details["pk3"]["is_pk"] is True + assert column_details["pk3"]["pk_position"] == 3 + assert column_details["content"]["is_pk"] is False + assert column_details["content"]["pk_position"] == 0 + + +def test_column_details_extra_defaults_and_notnull(): + with make_app_client(extra_databases={"defaults.db": """ + CREATE TABLE defaults ( + i INTEGER NOT NULL DEFAULT 42, + s TEXT DEFAULT 'hello', + dt TEXT DEFAULT (datetime('now')) + ); + """}) as client: + response = client.get("/defaults/defaults.json?_size=0&_extra=column_details") + assert response.status == 200 + column_details = response.json["column_details"] + assert column_details["i"]["notnull"] is True + assert column_details["i"]["default"] == "42" + assert column_details["s"]["notnull"] is False + assert column_details["s"]["default"] == "'hello'" + assert column_details["dt"]["default"] == "datetime('now')" + + +@pytest.mark.skipif( + sqlite_version() < (3, 31, 0), + reason="generated columns were added in SQLite 3.31.0", +) +def test_column_details_extra_generated_columns(): + with make_app_client(extra_databases={"generated.db": """ + CREATE TABLE generated_columns ( + body TEXT, + body_length_virtual INTEGER + GENERATED ALWAYS AS (length(body)) VIRTUAL, + body_length_stored INTEGER + GENERATED ALWAYS AS (length(body)) STORED + ); + """}) as client: + response = client.get( + "/generated/generated_columns.json?_size=0&_extra=column_details" + ) + assert response.status == 200 + column_details = response.json["column_details"] + assert column_details["body"]["hidden"] == 0 + assert column_details["body_length_virtual"]["hidden"] == 2 + assert column_details["body_length_stored"]["hidden"] == 3 + @pytest.mark.asyncio @pytest.mark.parametrize( From 19dde1c8604750a7613e2aa8e9500215dcb7ebb5 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 3 Jul 2026 16:09:27 -0700 Subject: [PATCH 4/7] Support BLOB values in row edit UI --- datasette/static/app.css | 112 ++++++++ datasette/static/edit-tools.js | 475 ++++++++++++++++++++++++++++++++- datasette/utils/__init__.py | 18 +- docs/binary_data.rst | 2 +- docs/changelog.rst | 2 + tests/test_playwright.py | 165 ++++++++++++ tests/test_utils.py | 6 +- 7 files changed, 754 insertions(+), 26 deletions(-) diff --git a/datasette/static/app.css b/datasette/static/app.css index ce800f61..5bfd0aca 100644 --- a/datasette/static/app.css +++ b/datasette/static/app.css @@ -1700,6 +1700,118 @@ textarea.row-edit-input { background: var(--paper); } +.row-edit-binary-control { + display: grid; + gap: 8px; + box-sizing: border-box; + width: 100%; + min-width: 0; + border: 1px solid var(--rule); + border-radius: 5px; + padding: 10px; + background: #fff; +} + +.row-edit-binary-control:focus { + border-color: var(--accent); + outline: 3px solid rgba(26, 86, 219, 0.12); +} + +.row-edit-binary-preview[hidden] { + display: none; +} + +.row-edit-binary-preview img { + display: block; + max-width: min(240px, 100%); + max-height: 180px; + border: 1px solid var(--rule); + border-radius: 4px; + background: var(--paper); +} + +.row-edit-binary-status { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 8px; + min-width: 0; +} + +.row-edit-binary-size { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 0.86rem; +} + +.row-edit-binary-name { + color: var(--muted); + font-size: 0.82rem; + overflow-wrap: anywhere; +} + +.row-edit-binary-name[hidden] { + display: none; +} + +.row-edit-binary-actions { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.row-edit-binary-file-button, +.row-edit-binary-clear { + appearance: none; + border: 1px solid var(--rule); + border-radius: 4px; + background: #fff; + color: var(--accent); + cursor: pointer; + font: inherit; + font-size: 0.78rem; + line-height: 1.2; + padding: 6px 8px; +} + +.row-edit-binary-file-button:hover, +.row-edit-binary-file-button:focus-within, +.row-edit-binary-clear:hover, +.row-edit-binary-clear:focus { + background: #f8fafc; +} + +.row-edit-binary-file-button:focus-within, +.row-edit-binary-clear:focus { + outline: 3px solid rgba(26, 86, 219, 0.12); + outline-offset: 1px; +} + +.row-edit-binary-file-button input[type="file"] { + position: absolute; + width: 1px; + height: 1px; + opacity: 0; + overflow: hidden; +} + +.row-edit-binary-clear[hidden] { + display: none; +} + +.row-edit-binary-drop-target { + border: 1px dashed var(--rule); + border-radius: 4px; + padding: 7px 8px; + color: var(--muted); + font-size: 0.78rem; +} + +.row-edit-binary-dragover .row-edit-binary-drop-target { + border-color: var(--accent); + background: var(--paper); + color: var(--ink); +} + .row-edit-default { display: grid; grid-template-columns: minmax(0, 1fr) 7.25rem; diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 9f4f89b9..b2891b5f 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -2,6 +2,7 @@ var ROW_DELETE_DIALOG_ID = "row-delete-dialog"; var rowDeleteDialogState = null; var ROW_EDIT_DIALOG_ID = "row-edit-dialog"; var rowEditDialogState = null; +var ROW_EDIT_BINARY_IMAGE_PREVIEW_MAX_BYTES = 10 * 1024 * 1024; var TABLE_CREATE_DIALOG_ID = "table-create-dialog"; var tableCreateDialogState = null; var TABLE_ALTER_DIALOG_ID = "table-alter-dialog"; @@ -3598,7 +3599,7 @@ function rowJsonUrl(row) { return ""; } url.pathname = url.pathname + ".json"; - url.searchParams.set("_extra", "columns,column_types"); + url.searchParams.set("_extra", "columns,column_types,column_details"); return url.toString(); } @@ -3878,10 +3879,155 @@ function initRowDeleteActions(manager) { }); } +function isBase64JsonValue(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return false; + } + var keys = Object.keys(value); + return ( + keys.length === 2 && + Object.prototype.hasOwnProperty.call(value, "$base64") && + Object.prototype.hasOwnProperty.call(value, "encoded") && + value.$base64 === true && + typeof value.encoded === "string" + ); +} + +function shouldUseBinaryControl(value, options) { + options = options || {}; + var sqliteType = (options.sqliteType || "").toLowerCase(); + return ( + isBase64JsonValue(value) || + sqliteType === "blob" || + options.valueKind === "binary" + ); +} + +function binaryEncodedValue(value) { + return isBase64JsonValue(value) ? value.encoded : ""; +} + +function binaryByteLengthFromBase64(encoded) { + encoded = (encoded || "").replace(/\s/g, ""); + if (!encoded) { + return 0; + } + var padding = 0; + if (encoded.slice(-2) === "==") { + padding = 2; + } else if (encoded.slice(-1) === "=") { + padding = 1; + } + return Math.max(0, Math.floor((encoded.length * 3) / 4) - padding); +} + +function rowEditBinaryValue(encoded) { + return { + $base64: true, + encoded: encoded || "", + }; +} + +function formatRowEditBinarySize(byteLength) { + var number = byteLength.toLocaleString(); + return "Binary: " + number + " byte" + (byteLength === 1 ? "" : "s"); +} + +function base64ToUint8Array(encoded) { + var binary = window.atob(encoded || ""); + var bytes = new Uint8Array(binary.length); + for (var i = 0; i < binary.length; i += 1) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +} + +function uint8ArrayToBase64(bytes) { + var chunks = []; + var chunkSize = 0x8000; + for (var i = 0; i < bytes.length; i += chunkSize) { + chunks.push( + String.fromCharCode.apply(null, bytes.subarray(i, i + chunkSize)), + ); + } + return window.btoa(chunks.join("")); +} + +function rowEditBinaryImageMimeType(bytes) { + if (!bytes || !bytes.length) { + return null; + } + if ( + bytes.length >= 8 && + bytes[0] === 0x89 && + bytes[1] === 0x50 && + bytes[2] === 0x4e && + bytes[3] === 0x47 && + bytes[4] === 0x0d && + bytes[5] === 0x0a && + bytes[6] === 0x1a && + bytes[7] === 0x0a + ) { + return "image/png"; + } + if ( + bytes.length >= 3 && + bytes[0] === 0xff && + bytes[1] === 0xd8 && + bytes[2] === 0xff + ) { + return "image/jpeg"; + } + if ( + bytes.length >= 6 && + bytes[0] === 0x47 && + bytes[1] === 0x49 && + bytes[2] === 0x46 && + bytes[3] === 0x38 && + (bytes[4] === 0x37 || bytes[4] === 0x39) && + bytes[5] === 0x61 + ) { + return "image/gif"; + } + if ( + bytes.length >= 12 && + bytes[0] === 0x52 && + bytes[1] === 0x49 && + bytes[2] === 0x46 && + bytes[3] === 0x46 && + bytes[8] === 0x57 && + bytes[9] === 0x45 && + bytes[10] === 0x42 && + bytes[11] === 0x50 + ) { + return "image/webp"; + } + if ( + bytes.length >= 12 && + bytes[4] === 0x66 && + bytes[5] === 0x74 && + bytes[6] === 0x79 && + bytes[7] === 0x70 && + bytes[8] === 0x61 && + bytes[9] === 0x76 && + bytes[10] === 0x69 && + (bytes[11] === 0x66 || bytes[11] === 0x73) + ) { + return "image/avif"; + } + if (bytes.length >= 2 && bytes[0] === 0x42 && bytes[1] === 0x4d) { + return "image/bmp"; + } + return null; +} + function valueToEditText(value) { if (value === null || typeof value === "undefined") { return ""; } + if (isBase64JsonValue(value)) { + return value.encoded; + } if (typeof value === "object") { return JSON.stringify(value, null, 2); } @@ -3892,6 +4038,9 @@ function shouldUseTextarea(value, columnType) { if (columnType && columnType.type === "textarea") { return true; } + if (isBase64JsonValue(value)) { + return false; + } if (value && typeof value === "object") { return true; } @@ -3900,6 +4049,9 @@ function shouldUseTextarea(value, columnType) { } function rowEditValueKind(value) { + if (isBase64JsonValue(value)) { + return "binary"; + } if (value === null || typeof value === "undefined") { return "null"; } @@ -3923,6 +4075,263 @@ function rowEditControlElement(control, autocompleteUrl) { return autocomplete; } +function revokeRowEditBinaryPreview(wrapper) { + if (wrapper && wrapper._rowEditBinaryPreviewUrl) { + URL.revokeObjectURL(wrapper._rowEditBinaryPreviewUrl); + wrapper._rowEditBinaryPreviewUrl = null; + } +} + +function updateRowEditBinaryPreview(wrapper, encoded, byteLength) { + var preview = wrapper.querySelector(".row-edit-binary-preview"); + if (!preview) { + return; + } + revokeRowEditBinaryPreview(wrapper); + preview.hidden = true; + preview.textContent = ""; + if ( + !encoded || + byteLength >= ROW_EDIT_BINARY_IMAGE_PREVIEW_MAX_BYTES || + !window.atob || + !window.Blob || + !window.URL || + !URL.createObjectURL + ) { + return; + } + + var bytes; + try { + bytes = base64ToUint8Array(encoded); + } catch (_error) { + return; + } + var mimeType = rowEditBinaryImageMimeType(bytes); + if (!mimeType) { + return; + } + + var image = document.createElement("img"); + image.alt = ""; + var objectUrl = URL.createObjectURL(new Blob([bytes], { type: mimeType })); + wrapper._rowEditBinaryPreviewUrl = objectUrl; + image.src = objectUrl; + preview.appendChild(image); + + var showPreview = function () { + if (wrapper._rowEditBinaryPreviewUrl === objectUrl) { + preview.hidden = false; + } + }; + var hidePreview = function () { + if (wrapper._rowEditBinaryPreviewUrl === objectUrl) { + revokeRowEditBinaryPreview(wrapper); + preview.hidden = true; + preview.textContent = ""; + } + }; + if (image.decode) { + image.decode().then(showPreview).catch(hidePreview); + } else { + image.onload = showPreview; + image.onerror = hidePreview; + } +} + +function updateRowEditBinaryDisplay(wrapper, control, fileName) { + var kind = rowEditControlValueKind(control); + var size = wrapper.querySelector(".row-edit-binary-size"); + var name = wrapper.querySelector(".row-edit-binary-name"); + var clear = wrapper.querySelector(".row-edit-binary-clear"); + var byteLength = + kind === "binary" ? binaryByteLengthFromBase64(control.value) : null; + + if (size) { + size.textContent = + kind === "binary" ? formatRowEditBinarySize(byteLength) : "No binary data"; + } + if (name) { + name.textContent = fileName || ""; + name.hidden = !fileName; + } + if (clear) { + clear.hidden = control.dataset.notNull === "1" || kind !== "binary"; + } + if (kind === "binary") { + updateRowEditBinaryPreview(wrapper, control.value, byteLength); + } else { + updateRowEditBinaryPreview(wrapper, "", 0); + } +} + +function setRowEditBinaryControlValue(control, encoded, fileName) { + control.value = encoded || ""; + control.dataset.currentValueKind = "binary"; + updateRowEditBinaryDisplay(control._rowEditBinaryWrapper, control, fileName); + control.dispatchEvent(new Event("input", { bubbles: true })); +} + +function clearRowEditBinaryControlValue(control) { + control.value = ""; + control.dataset.currentValueKind = "null"; + updateRowEditBinaryDisplay(control._rowEditBinaryWrapper, control, ""); + control.dispatchEvent(new Event("input", { bubbles: true })); +} + +function readRowEditBinaryFile(file) { + return new Promise(function (resolve, reject) { + var reader = new FileReader(); + reader.onload = function () { + try { + var bytes = new Uint8Array(reader.result || new ArrayBuffer(0)); + resolve({ + encoded: uint8ArrayToBase64(bytes), + name: file && file.name ? file.name : "", + }); + } catch (error) { + reject(error); + } + }; + reader.onerror = function () { + reject(reader.error || new Error("Could not read file")); + }; + reader.readAsArrayBuffer(file); + }); +} + +function rowEditBinaryValueFromText(text) { + var encoder = new TextEncoder(); + var bytes = encoder.encode(text || ""); + return { + encoded: uint8ArrayToBase64(bytes), + name: "Pasted text", + }; +} + +function rowEditBinaryFirstClipboardFile(clipboardData) { + if (!clipboardData) { + return null; + } + if (clipboardData.files && clipboardData.files.length) { + return clipboardData.files[0]; + } + var items = clipboardData.items || []; + for (var i = 0; i < items.length; i += 1) { + if (items[i].kind === "file") { + return items[i].getAsFile(); + } + } + return null; +} + +function handleRowEditBinaryFile(control, file) { + if (!file) { + return; + } + readRowEditBinaryFile(file) + .then(function (value) { + setRowEditBinaryControlValue(control, value.encoded, value.name); + }) + .catch(function (error) { + console.error("Could not read binary file", error); + }); +} + +function createRowEditBinaryControlElement(control, value, options, labelId) { + var wrapper = document.createElement("div"); + wrapper.className = "row-edit-binary-control"; + wrapper.dataset.column = control.name; + wrapper.setAttribute("role", "group"); + wrapper.setAttribute("tabindex", "0"); + wrapper.setAttribute("aria-labelledby", labelId); + + var preview = document.createElement("div"); + preview.className = "row-edit-binary-preview"; + preview.hidden = true; + + var status = document.createElement("div"); + status.className = "row-edit-binary-status"; + var size = document.createElement("span"); + size.className = "row-edit-binary-size"; + var name = document.createElement("span"); + name.className = "row-edit-binary-name"; + name.hidden = true; + status.appendChild(size); + status.appendChild(name); + + var actions = document.createElement("div"); + actions.className = "row-edit-binary-actions"; + var fileLabel = document.createElement("label"); + fileLabel.className = "row-edit-binary-file-button"; + fileLabel.textContent = "Attach file"; + var fileInput = document.createElement("input"); + fileInput.type = "file"; + fileInput.setAttribute("aria-label", "Attach file for " + control.name); + fileInput.addEventListener("change", function () { + if (fileInput.files && fileInput.files.length) { + handleRowEditBinaryFile(control, fileInput.files[0]); + } + }); + fileLabel.appendChild(fileInput); + actions.appendChild(fileLabel); + + var clearButton = document.createElement("button"); + clearButton.type = "button"; + clearButton.className = "row-edit-binary-clear"; + clearButton.textContent = "Set NULL"; + clearButton.addEventListener("click", function () { + clearRowEditBinaryControlValue(control); + wrapper.focus(); + }); + actions.appendChild(clearButton); + + var dropTarget = document.createElement("div"); + dropTarget.className = "row-edit-binary-drop-target"; + dropTarget.textContent = "Drop or paste file contents"; + ["dragenter", "dragover"].forEach(function (eventName) { + wrapper.addEventListener(eventName, function (ev) { + ev.preventDefault(); + wrapper.classList.add("row-edit-binary-dragover"); + }); + }); + ["dragleave", "drop"].forEach(function (eventName) { + wrapper.addEventListener(eventName, function () { + wrapper.classList.remove("row-edit-binary-dragover"); + }); + }); + wrapper.addEventListener("drop", function (ev) { + ev.preventDefault(); + var file = ev.dataTransfer && ev.dataTransfer.files[0]; + handleRowEditBinaryFile(control, file); + }); + wrapper.addEventListener("paste", function (ev) { + var file = rowEditBinaryFirstClipboardFile(ev.clipboardData); + if (file) { + ev.preventDefault(); + handleRowEditBinaryFile(control, file); + return; + } + var text = ev.clipboardData && ev.clipboardData.getData("text"); + if (text) { + ev.preventDefault(); + var pasted = rowEditBinaryValueFromText(text); + setRowEditBinaryControlValue(control, pasted.encoded, pasted.name); + } + }); + + control.type = "hidden"; + control._rowEditBinaryWrapper = wrapper; + wrapper.appendChild(control); + wrapper.appendChild(preview); + wrapper.appendChild(status); + wrapper.appendChild(actions); + wrapper.appendChild(dropTarget); + + updateRowEditBinaryDisplay(wrapper, control, ""); + return wrapper; +} + function columnTypeForContext(columnType) { if (!columnType) { return null; @@ -4186,6 +4595,11 @@ function focusFirstRowEditControl(state, options) { if (focusRowEditPluginControl(field)) { return true; } + var binaryControl = field.querySelector(".row-edit-binary-control"); + if (binaryControl) { + binaryControl.focus(); + return true; + } control.focus(); return true; } @@ -4209,6 +4623,11 @@ function destroyRowEditFields(state) { } } }); + state.fields + .querySelectorAll(".row-edit-binary-control") + .forEach(function (binaryControl) { + revokeRowEditBinaryPreview(binaryControl); + }); state.fields.innerHTML = ""; } @@ -4235,34 +4654,50 @@ function createRowEditField(column, value, isPk, columnType, index, options) { controlWrap.className = "row-edit-control-wrap"; var context = columnFormControlContext(column, isPk, columnType, options); - var pluginControl = makeColumnField(options.manager, context); + var isBinaryField = shouldUseBinaryControl(value, options); + var pluginControl = isBinaryField + ? null + : makeColumnField(options.manager, context); var useTextarea = - (pluginControl && pluginControl.useTextarea === true) || - shouldUseTextarea(value, columnType); + !isBinaryField && + ((pluginControl && pluginControl.useTextarea === true) || + shouldUseTextarea(value, columnType)); var control = useTextarea ? document.createElement("textarea") : document.createElement("input"); + var initialValue = isBinaryField + ? binaryEncodedValue(value) + : valueToEditText(value); + var initialValueKind = options.valueKind || rowEditValueKind(value); + if (isBinaryField) { + initialValueKind = isBase64JsonValue(value) ? "binary" : "null"; + } control.className = "row-edit-input"; control.id = fieldId; control.name = column; - control.value = valueToEditText(value); + control.value = initialValue; control.setAttribute("aria-describedby", metaId); - control.dataset.initialValue = valueToEditText(value); - control.dataset.initialValueKind = - options.valueKind || rowEditValueKind(value); + control.dataset.initialValue = initialValue; + control.dataset.initialValueKind = initialValueKind; control.dataset.primaryKey = isPk ? "1" : "0"; control.dataset.currentValueKind = control.dataset.initialValueKind; + if (isBinaryField) { + control.dataset.binaryField = "1"; + control.dataset.notNull = options.notnull ? "1" : "0"; + } if (hasDefaultExpression) { control.dataset.useSqliteDefault = useSqliteDefault ? "1" : "0"; } if (useSqliteDefault) { control.disabled = true; } - if (options.omitIfBlank) { + if (options.omitIfBlank || (isBinaryField && options.mode === "insert")) { control.dataset.omitIfBlank = "1"; } - if (control.nodeName === "TEXTAREA") { + if (isBinaryField) { + control.type = "hidden"; + } else if (control.nodeName === "TEXTAREA") { control.rows = Math.min(8, Math.max(3, control.value.split("\n").length)); } else { control.type = "text"; @@ -4336,9 +4771,11 @@ function createRowEditField(column, value, isPk, columnType, index, options) { field._datasetteColumnFormField = fieldApi; var pluginControlElement = renderColumnField(pluginControl, fieldApi); var controlElement = + (isBinaryField && + createRowEditBinaryControlElement(control, value, options, labelId)) || pluginControlElement || rowEditControlElement(control, options.autocompleteUrl); - if (options.autocompleteUrl && !pluginControlElement) { + if (options.autocompleteUrl && !pluginControlElement && !isBinaryField) { control.addEventListener("input", function () { setForeignKeyMetaLink(meta, options.autocompleteUrl, null); }); @@ -4460,6 +4897,9 @@ function setRowEditDialogSaving(state, isSaving) { function valueFromRowEditControl(control) { var value = control.value; + if (rowEditControlValueKind(control) === "binary") { + return rowEditBinaryValue(value); + } return valueFromRowEditText( control.name, value, @@ -4470,6 +4910,9 @@ function valueFromRowEditControl(control) { function valueFromRowEditText(name, value, initialValueKind) { var trimmed = value.trim(); + if (initialValueKind === "binary") { + return rowEditBinaryValue(value); + } if (initialValueKind === "null" && value === "") { return null; } @@ -4597,7 +5040,11 @@ function collectRowFormValues(state) { if (control.dataset.useSqliteDefault === "1") { return; } - if (control.dataset.omitIfBlank === "1" && control.value === "") { + if ( + control.dataset.omitIfBlank === "1" && + control.value === "" && + rowEditControlValueKind(control) !== "binary" + ) { return; } if ( @@ -4919,9 +5366,11 @@ function renderRowEditFields(state, data) { var columns = data.columns || (row ? Object.keys(row) : []); var primaryKeys = data.primary_keys || []; var columnTypes = data.column_types || {}; + var columnDetails = data.column_details || {}; destroyRowEditFields(state); columns.forEach(function (column, index) { + var columnDetail = columnDetails[column] || {}; state.fields.appendChild( createRowEditField( column, @@ -4935,7 +5384,9 @@ function renderRowEditFields(state, data) { form: state.form, manager: state.manager, mode: state.mode, + notnull: columnDetail.notnull, primaryKeyReadonly: true, + sqliteType: columnDetail.sqlite_type, }, ), ); diff --git a/datasette/utils/__init__.py b/datasette/utils/__init__.py index 468828c5..dfd5440c 100644 --- a/datasette/utils/__init__.py +++ b/datasette/utils/__init__.py @@ -237,10 +237,8 @@ class CustomJSONEncoder(json.JSONEncoder): - ``sqlite3.Row`` becomes a tuple - ``sqlite3.Cursor`` becomes a list - If a binary blob can be decoded as UTF-8, the encoder returns it as text. - - If it can't (for example, images), it is encoded as an object, with the actual - data base64-encoded, like so: :: + Binary blobs are encoded as an object, with the actual data base64-encoded, + like so: :: { "$base64": True, @@ -256,14 +254,10 @@ class CustomJSONEncoder(json.JSONEncoder): if isinstance(obj, sqlite3.Cursor): return list(obj) if isinstance(obj, bytes): - # Does it encode to utf8? - try: - return obj.decode("utf8") - except UnicodeDecodeError: - return { - "$base64": True, - "encoded": base64.b64encode(obj).decode("latin1"), - } + return { + "$base64": True, + "encoded": base64.b64encode(obj).decode("latin1"), + } return json.JSONEncoder.default(self, obj) diff --git a/docs/binary_data.rst b/docs/binary_data.rst index fc972f98..d41e36be 100644 --- a/docs/binary_data.rst +++ b/docs/binary_data.rst @@ -17,7 +17,7 @@ Datasette includes special handling for these binary values. The Datasette inter Binary values in JSON --------------------- -Binary data is represented in ``.json`` exports using Base64 encoding. +Binary data is represented in ``.json`` exports using Base64 encoding. Datasette uses this representation for every ``BLOB`` value, including binary values that could also be decoded as UTF-8 text. https://latest.datasette.io/fixtures/binary_data.json?_shape=array diff --git a/docs/changelog.rst b/docs/changelog.rst index 7f3fd2c4..0cf1ec0b 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -9,6 +9,8 @@ Changelog 1.0a36 (in development) ----------------------- +- Datasette's JSON APIs now consistently encode every ``BLOB`` value using the documented :ref:`binary value JSON format `, even when the bytes could be decoded as UTF-8 text. +- The insert and edit row dialogs now provide a dedicated control for ``BLOB`` values. Existing binary values are shown by byte size, image values under 10MB are previewed as thumbnails, and replacements can be attached, dropped or pasted into the control. - The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata. .. _v1_0_a35: diff --git a/tests/test_playwright.py b/tests/test_playwright.py index ee396de5..63e06b29 100644 --- a/tests/test_playwright.py +++ b/tests/test_playwright.py @@ -1,3 +1,4 @@ +import base64 import json import socket import subprocess @@ -10,6 +11,10 @@ import pytest from datasette.fixtures import write_fixture_database from datasette.utils.sqlite import sqlite3 +PNG_1X1_BYTES = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=" +) + def find_free_port(): with socket.socket() as sock: @@ -102,6 +107,11 @@ def write_playwright_database(db_path): id integer primary key, created_ms integer default (CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)) ); + create table binary_files ( + id integer primary key, + name text not null, + data blob + ); insert into projects (title, metadata, logo, notes, score) values ( 'Build Datasette', @@ -111,6 +121,19 @@ def write_playwright_database(db_path): 5 ); """) + conn.execute( + "insert into binary_files (name, data) values (?, ?)", + ("Raw bytes", b"\x00\x01\x02\x03"), + ) + conn.execute( + "insert into binary_files (name, data) values (?, ?)", + ("PNG image", PNG_1X1_BYTES), + ) + conn.execute( + "insert into binary_files (name, data) values (?, ?)", + ("Null bytes", None), + ) + conn.commit() finally: conn.close() @@ -145,6 +168,14 @@ def write_playwright_config(config_path): "alter-table": True, }, }, + "binary_files": { + "label_column": "name", + "permissions": { + "insert-row": True, + "update-row": True, + "delete-row": True, + }, + }, }, }, }, @@ -278,6 +309,15 @@ def project_row(datasette_server, pk): return rows[0] +def binary_file_blob(datasette_server, pk): + response = httpx.get( + f"{datasette_server}data/binary_files/{pk}.blob", + params={"_blob_column": "data"}, + ) + response.raise_for_status() + return response.content + + def open_jump_menu(page): page.keyboard.press("/") page.locator("navigation-search .search-input").wait_for() @@ -877,6 +917,131 @@ def test_edit_row_flow_validates_json_and_saves_changes(page, datasette_server): assert data["notes"] == "Edited from Playwright" +@pytest.mark.playwright +def test_edit_row_binary_control_shows_size_and_image_preview(page, datasette_server): + page.goto(f"{datasette_server}data/binary_files") + page.locator('tr[data-row="1"] button[data-row-action="edit"]').click() + + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + raw_control = dialog.locator('.row-edit-binary-control[data-column="data"]') + raw_control.wait_for() + assert ( + raw_control.locator(".row-edit-binary-size").inner_text() == "Binary: 4 bytes" + ) + assert raw_control.locator(".row-edit-binary-preview img").count() == 0 + assert dialog.locator('textarea[name="data"]').count() == 0 + assert dialog.locator('input[type="hidden"][name="data"]').count() == 1 + + dialog.locator(".row-edit-cancel").click() + page.locator('tr[data-row="2"] button[data-row-action="edit"]').click() + + image_control = dialog.locator('.row-edit-binary-control[data-column="data"]') + image_control.wait_for() + assert ( + image_control.locator(".row-edit-binary-size").inner_text() + == f"Binary: {len(PNG_1X1_BYTES)} bytes" + ) + image = image_control.locator(".row-edit-binary-preview img") + image.wait_for() + assert image.get_attribute("src").startswith("blob:") + + +@pytest.mark.playwright +def test_edit_row_binary_control_replaces_blob_from_file(page, datasette_server): + replacement = b"Replacement \x00 bytes" + + page.goto(f"{datasette_server}data/binary_files") + page.locator('tr[data-row="1"] button[data-row-action="edit"]').click() + + dialog = page.locator("#row-edit-dialog") + binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]') + binary_control.wait_for() + binary_control.locator('input[type="file"]').set_input_files( + { + "name": "replacement.bin", + "mimeType": "application/octet-stream", + "buffer": replacement, + } + ) + assert ( + binary_control.locator(".row-edit-binary-size").inner_text() + == f"Binary: {len(replacement)} bytes" + ) + assert "replacement.bin" in binary_control.inner_text() + + dialog.locator(".row-edit-save").click() + page.locator(".row-mutation-status", has_text="Updated row 1").wait_for() + assert binary_file_blob(datasette_server, 1) == replacement + + +@pytest.mark.playwright +def test_edit_row_binary_control_handles_null_blob(page, datasette_server): + replacement = b"From NULL" + + page.goto(f"{datasette_server}data/binary_files") + page.locator('tr[data-row="3"] button[data-row-action="edit"]').click() + + dialog = page.locator("#row-edit-dialog") + binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]') + binary_control.wait_for() + assert ( + binary_control.locator(".row-edit-binary-size").inner_text() == "No binary data" + ) + binary_control.locator('input[type="file"]').set_input_files( + { + "name": "from-null.bin", + "mimeType": "application/octet-stream", + "buffer": replacement, + } + ) + assert ( + binary_control.locator(".row-edit-binary-size").inner_text() + == f"Binary: {len(replacement)} bytes" + ) + + dialog.locator(".row-edit-save").click() + page.locator(".row-mutation-status", has_text="Updated row 3").wait_for() + assert binary_file_blob(datasette_server, 3) == replacement + + +@pytest.mark.playwright +def test_insert_row_binary_control_accepts_pasted_file(page, datasette_server): + pasted = b"Pasted \x00 bytes" + + page.goto(f"{datasette_server}data/binary_files") + page.locator('button[data-table-action="insert-row"]').click() + + dialog = page.locator("#row-edit-dialog") + dialog.wait_for() + dialog.locator('input[name="name"]').fill("Pasted bytes") + binary_control = dialog.locator('.row-edit-binary-control[data-column="data"]') + binary_control.wait_for() + binary_control.evaluate( + """(node, bytes) => { + const transfer = new DataTransfer(); + transfer.items.add( + new File([new Uint8Array(bytes)], "pasted.bin", { + type: "application/octet-stream" + }) + ); + const event = new Event("paste", { bubbles: true, cancelable: true }); + Object.defineProperty(event, "clipboardData", { value: transfer }); + node.dispatchEvent(event); + }""", + list(pasted), + ) + assert ( + binary_control.locator(".row-edit-binary-size").inner_text() + == f"Binary: {len(pasted)} bytes" + ) + assert "pasted.bin" in binary_control.inner_text() + + dialog.locator(".row-edit-save").click() + page.locator(".row-mutation-status", has_text="Inserted row 4").wait_for() + assert binary_file_blob(datasette_server, 4) == pasted + + @pytest.mark.playwright def test_delete_row_flow_removes_row(page, datasette_server): page.goto(f"{datasette_server}data/projects") diff --git a/tests/test_utils.py b/tests/test_utils.py index 38ebb51e..46dcd89d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -132,7 +132,11 @@ def test_path_from_row_pks(row, pks, expected_path): """ {"CategoryID": 1, "Description": "Soft drinks", "Picture": {"$base64": true, "encoded": "FRwCx60F/g=="}} """.strip(), - ) + ), + ( + {"message": b"hello"}, + '{"message": {"$base64": true, "encoded": "aGVsbG8="}}', + ), ], ) def test_custom_json_encoder(obj, expected): From 5f05d33ef70e8fb8bf5694ae1dab1f6c14719051 Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 3 Jul 2026 16:44:07 -0700 Subject: [PATCH 5/7] Run Prettier --- datasette/static/edit-tools.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index b2891b5f..89cf5f94 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -4149,7 +4149,9 @@ function updateRowEditBinaryDisplay(wrapper, control, fileName) { if (size) { size.textContent = - kind === "binary" ? formatRowEditBinarySize(byteLength) : "No binary data"; + kind === "binary" + ? formatRowEditBinarySize(byteLength) + : "No binary data"; } if (name) { name.textContent = fileName || ""; From 81d6ee69cd558de3ba589583a11a27ad25f9ea1b Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 3 Jul 2026 17:12:26 -0700 Subject: [PATCH 6/7] Allow declared column type case to vary --- tests/test_api.py | 32 +++++++++++---------- tests/test_table_api.py | 61 +++++++++++++++++++++-------------------- 2 files changed, 50 insertions(+), 43 deletions(-) diff --git a/tests/test_api.py b/tests/test_api.py index ee3329e1..d99c3341 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -446,8 +446,10 @@ async def test_row_extras(ds_client): "format": "json", } assert len(data["foreign_key_tables"]) == 5 - assert data["column_details"]["id"] == { - "type": "INTEGER", + id_detail = data["column_details"]["id"] + assert id_detail["type"].lower() == "integer" + assert id_detail == { + "type": id_detail["type"], "sqlite_type": "INTEGER", "notnull": False, "default": None, @@ -455,8 +457,10 @@ async def test_row_extras(ds_client): "pk_position": 1, "hidden": 0, } - assert data["column_details"]["content"] == { - "type": "TEXT", + content_detail = data["column_details"]["content"] + assert content_detail["type"].lower() == "text" + assert content_detail == { + "type": content_detail["type"], "sqlite_type": "TEXT", "notnull": False, "default": None, @@ -470,16 +474,16 @@ async def test_row_extras(ds_client): async def test_column_details_extra_row_for_null_blob(ds_client): response = await ds_client.get("/fixtures/binary_data/3.json?_extra=column_details") assert response.status_code == 200 - assert response.json()["column_details"] == { - "data": { - "type": "BLOB", - "sqlite_type": "BLOB", - "notnull": False, - "default": None, - "is_pk": False, - "pk_position": 0, - "hidden": 0, - } + data_detail = response.json()["column_details"]["data"] + assert data_detail["type"].lower() == "blob" + assert data_detail == { + "type": data_detail["type"], + "sqlite_type": "BLOB", + "notnull": False, + "default": None, + "is_pk": False, + "pk_position": 0, + "hidden": 0, } diff --git a/tests/test_table_api.py b/tests/test_table_api.py index 067a8bab..005a4680 100644 --- a/tests/test_table_api.py +++ b/tests/test_table_api.py @@ -1342,41 +1342,44 @@ async def test_column_details_extra_table(ds_client): "/fixtures/binary_data.json?_size=0&_extra=column_details" ) assert response.status_code == 200 - assert response.json()["column_details"] == { - "data": { - "type": "BLOB", - "sqlite_type": "BLOB", - "notnull": False, - "default": None, - "is_pk": False, - "pk_position": 0, - "hidden": 0, - } + data_detail = response.json()["column_details"]["data"] + assert data_detail["type"].lower() == "blob" + assert data_detail == { + "type": data_detail["type"], + "sqlite_type": "BLOB", + "notnull": False, + "default": None, + "is_pk": False, + "pk_position": 0, + "hidden": 0, } response = await ds_client.get( "/fixtures/simple_primary_key.json?_size=0&_extra=column_details" ) assert response.status_code == 200 - assert response.json()["column_details"] == { - "id": { - "type": "INTEGER", - "sqlite_type": "INTEGER", - "notnull": False, - "default": None, - "is_pk": True, - "pk_position": 1, - "hidden": 0, - }, - "content": { - "type": "TEXT", - "sqlite_type": "TEXT", - "notnull": False, - "default": None, - "is_pk": False, - "pk_position": 0, - "hidden": 0, - }, + column_details = response.json()["column_details"] + id_detail = column_details["id"] + assert id_detail["type"].lower() == "integer" + assert id_detail == { + "type": id_detail["type"], + "sqlite_type": "INTEGER", + "notnull": False, + "default": None, + "is_pk": True, + "pk_position": 1, + "hidden": 0, + } + content_detail = column_details["content"] + assert content_detail["type"].lower() == "text" + assert content_detail == { + "type": content_detail["type"], + "sqlite_type": "TEXT", + "notnull": False, + "default": None, + "is_pk": False, + "pk_position": 0, + "hidden": 0, } response = await ds_client.get( From ae26c3372aa550aee515db831ca8d3771902473f Mon Sep 17 00:00:00 2001 From: Simon Willison Date: Fri, 3 Jul 2026 17:19:13 -0700 Subject: [PATCH 7/7] Use image load events for BLOB previews --- datasette/static/edit-tools.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/datasette/static/edit-tools.js b/datasette/static/edit-tools.js index 89cf5f94..2aa9adcf 100644 --- a/datasette/static/edit-tools.js +++ b/datasette/static/edit-tools.js @@ -4116,8 +4116,6 @@ function updateRowEditBinaryPreview(wrapper, encoded, byteLength) { image.alt = ""; var objectUrl = URL.createObjectURL(new Blob([bytes], { type: mimeType })); wrapper._rowEditBinaryPreviewUrl = objectUrl; - image.src = objectUrl; - preview.appendChild(image); var showPreview = function () { if (wrapper._rowEditBinaryPreviewUrl === objectUrl) { @@ -4131,12 +4129,10 @@ function updateRowEditBinaryPreview(wrapper, encoded, byteLength) { preview.textContent = ""; } }; - if (image.decode) { - image.decode().then(showPreview).catch(hidePreview); - } else { - image.onload = showPreview; - image.onerror = hidePreview; - } + image.onload = showPreview; + image.onerror = hidePreview; + image.src = objectUrl; + preview.appendChild(image); } function updateRowEditBinaryDisplay(wrapper, control, fileName) {