Support editing BLOBs through JSON APIs

datasette/utils/__init__.py

@@ -1,4 +1,5 @@
 import asyncio
+import binascii
 from contextlib import contextmanager
 import aiofiles
 import click
@@ -266,6 +267,35 @@ class CustomJSONEncoder(json.JSONEncoder):
         return json.JSONEncoder.default(self, obj)
 
 
+class WriteJsonValueError(ValueError):
+    pass
+
+
+def decode_write_json_cell(value):
+    if not isinstance(value, dict):
+        return value
+    keys = set(value.keys())
+    if keys == {"$raw"}:
+        return value["$raw"]
+    if keys == {"$base64", "encoded"} and value.get("$base64") is True:
+        encoded = value["encoded"]
+        if not isinstance(encoded, str):
+            raise WriteJsonValueError("$base64 encoded value must be a string")
+        try:
+            return base64.b64decode(encoded, validate=True)
+        except binascii.Error as ex:
+            raise WriteJsonValueError("Invalid $base64 encoded value") from ex
+    return value
+
+
+def decode_write_json_row(row):
+    return {key: decode_write_json_cell(value) for key, value in row.items()}
+
+
+def decode_write_json_rows(rows):
+    return [decode_write_json_row(row) for row in rows]
+
+
 @contextmanager
 def sqlite_timelimit(conn, ms):
     deadline = time.perf_counter() + (ms / 1000)

datasette/views/row.py

@@ -17,7 +17,9 @@ from datasette.utils import (
     add_cors_headers,
     await_me_maybe,
     call_with_supported_arguments,
+    CustomJSONEncoder,
     CustomRow,
+    decode_write_json_row,
     InvalidSql,
     make_slot_function,
     path_from_row_pks,
@@ -27,6 +29,7 @@ from datasette.utils import (
     to_css_class,
     escape_sqlite,
     sqlite3,
+    WriteJsonValueError,
 )
 from datasette.plugins import pm
 from datasette.extras import extra_names_from_request, ExtraScope
@@ -806,6 +809,10 @@ class RowUpdateView(BaseView):
             return _error(["Invalid keys: {}".format(", ".join(invalid_keys))])
 
         update = data["update"]
+        try:
+            update = decode_write_json_row(update)
+        except WriteJsonValueError as e:
+            return _error([str(e)], 400)
 
         # Validate column types
         from datasette.views.table import _validate_column_types
@@ -867,4 +874,4 @@ class RowUpdateView(BaseView):
                 self.ds.INFO,
             )
 
-        return Response.json(result, status=200)
+        return Response.json(result, status=200, default=CustomJSONEncoder().default)

datasette/views/table.py

@@ -22,9 +22,11 @@ from datasette.utils import (
     add_cors_headers,
     await_me_maybe,
     call_with_supported_arguments,
+    CustomJSONEncoder,
     CustomRow,
     append_querystring,
     compound_keys_after_sql,
+    decode_write_json_rows,
     format_bytes,
     make_slot_function,
     tilde_encode,
@@ -41,6 +43,7 @@ from datasette.utils import (
     urlsafe_components,
     value_as_boolean,
     InvalidSql,
+    WriteJsonValueError,
     sqlite3,
 )
 from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response
@@ -1057,6 +1060,10 @@ class TableInsertView(BaseView):
         )
         if errors:
             return _error(errors, 400)
+        try:
+            rows = decode_write_json_rows(rows)
+        except WriteJsonValueError as e:
+            return _error([str(e)], 400)
 
         # Validate column types
         ct_errors = await _validate_column_types(
@@ -1191,7 +1198,11 @@ class TableInsertView(BaseView):
                     )
                 )
 
-        return Response.json(result, status=200 if upsert else 201)
+        return Response.json(
+            result,
+            status=200 if upsert else 201,
+            default=CustomJSONEncoder().default,
+        )
 
 
 class TableUpsertView(TableInsertView):

datasette/views/table_create_alter.py

@@ -20,9 +20,11 @@ from datasette.column_types import SQLiteType
 from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent
 from datasette.resources import DatabaseResource, TableResource
 from datasette.utils import (
+    decode_write_json_rows,
     escape_sqlite,
     get_outbound_foreign_keys,
     table_column_details,
+    WriteJsonValueError,
 )
 from datasette.utils.asgi import NotFound, Response
 from datasette.utils.sqlite import sqlite_hidden_table_names
@@ -838,6 +840,10 @@ class TableCreateView(BaseView):
                 actor=request.actor,
             ):
                 return _error(["Permission denied: need insert-row"], 403)
+            try:
+                rows = decode_write_json_rows(rows)
+            except WriteJsonValueError as e:
+                return _error([str(e)], 400)
 
         alter = False
         if rows:

docs/binary_data.rst

@@ -12,6 +12,11 @@ Datasette includes special handling for these binary values. The Datasette inter
    :width: 311px
    :alt: Screenshot showing download links next to binary data in the table view
 
+.. _binary_json_format:
+
+Binary values in JSON
+---------------------
+
 Binary data is represented in ``.json`` exports using Base64 encoding.
 
 https://latest.datasette.io/fixtures/binary_data.json?_shape=array
@@ -39,6 +44,48 @@ https://latest.datasette.io/fixtures/binary_data.json?_shape=array
         }
     ]
 
+The same format can be used with the :ref:`JSON write API <json_api_write>`.
+If a column value in a ``row``, ``rows`` or ``update`` object is a JSON object with exactly ``"$base64"`` set to ``true`` and an ``"encoded"`` string, Datasette will decode that Base64 string and store the resulting bytes:
+
+.. code-block:: json
+
+    {
+        "data": {
+            "$base64": true,
+            "encoded": "FRwCx60F/g=="
+        }
+    }
+
+This works for inserts, upserts and updates. It also works when creating a table from example ``row`` or ``rows`` data: Datasette decodes the value before inferring the schema, allowing that column to be created as a ``BLOB`` column.
+
+To store a JSON object with that exact shape literally, wrap it in a ``"$raw"`` object:
+
+.. code-block:: json
+
+    {
+        "data": {
+            "$raw": {
+                "$base64": true,
+                "encoded": "FRwCx60F/g=="
+            }
+        }
+    }
+
+``"$raw"`` unwraps exactly one layer. To store a literal ``"$raw"`` object containing a Base64 object, wrap it again:
+
+.. code-block:: json
+
+    {
+        "data": {
+            "$raw": {
+                "$raw": {
+                    "$base64": true,
+                    "encoded": "FRwCx60F/g=="
+                }
+            }
+        }
+    }
+
 .. _binary_linking:
 
 Linking to binary downloads

docs/json_api.rst

@@ -1532,6 +1532,8 @@ The JSON write API
 
 Datasette provides a write API for JSON data. This is a POST-only API that requires an authenticated API token, see :ref:`CreateTokenView`. The token will need to have the specified :ref:`authentication_permissions`.
 
+The row-based write APIs can write :ref:`binary values in JSON <binary_json_format>` using Datasette's Base64 representation for BLOB data.
+
 .. _ExecuteWriteView:
 
 Executing write SQL
@@ -1660,6 +1662,8 @@ A single row can be inserted using the ``"row"`` key:
         }
     }
 
+Column values can use the :ref:`binary value JSON format <binary_json_format>` to write BLOB data.
+
 If successful, this will return a ``201`` status code and the newly inserted row, for example:
 
 .. code-block:: json
@@ -1765,6 +1769,8 @@ An upsert is an insert or update operation. If a row with a matching primary key
 
 The upsert API is mostly the same shape as the :ref:`insert API <TableInsertView>`. It requires both the :ref:`actions_insert_row` and :ref:`actions_update_row` permissions.
 
+It also accepts the same :ref:`binary value JSON format <binary_json_format>`.
+
 ::
 
     POST /<database>/<table>/-/upsert
@@ -1895,6 +1901,8 @@ To update a row, make a ``POST`` to ``/<database>/<table>/<row-pks>/-/update``.
 
 You only need to pass the columns you want to update. Any other columns will be left unchanged.
 
+Updated values can use the :ref:`binary value JSON format <binary_json_format>`.
+
 If successful, this will return a ``200`` status code and a ``{"ok": true}`` response body.
 
 Add ``"return": true`` to the request body to return the updated row:
@@ -2098,6 +2106,8 @@ Datasette will create a table with a schema that matches those rows and insert t
         "pk": "id"
     }
 
+Example rows can use the :ref:`binary value JSON format <binary_json_format>`, allowing Datasette to infer ``BLOB`` columns.
+
 Doing this requires both the :ref:`actions_create_table` and :ref:`actions_insert_row` permissions.
 
 The ``201`` response here will be similar to the ``columns`` form, but will also include the number of rows that were inserted as ``row_count``:

tests/test_api_write.py

@@ -50,6 +50,133 @@ def _insert_and_fetch_created(conn, table, insert_sql):
     ).fetchone()
 
 
+BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"}
+BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}'
+
+
+@pytest.mark.asyncio
+async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write):
+    token = write_token(ds_write)
+    response = await ds_write.client.post(
+        "/data/-/create",
+        json={
+            "table": "binary_create",
+            "row": {
+                "id": 1,
+                "data": BASE64_WRITE_API_VALUE,
+                "literal": {"$raw": BASE64_WRITE_API_VALUE},
+                "double_raw": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}},
+            },
+            "pk": "id",
+        },
+        headers=_headers(token),
+    )
+    assert response.status_code == 201
+    assert "[data] BLOB" in response.json()["schema"]
+    assert "[literal] TEXT" in response.json()["schema"]
+
+    rows = (await ds_write.get_database("data").execute("""
+            select
+              typeof(data) as data_type,
+              hex(data) as data_hex,
+              typeof(literal) as literal_type,
+              literal,
+              typeof(double_raw) as double_raw_type,
+              double_raw
+            from binary_create
+            """)).dicts()
+    assert rows == [
+        {
+            "data_type": "blob",
+            "data_hex": "000102FDFEFF",
+            "literal_type": "text",
+            "literal": BASE64_WRITE_API_LITERAL,
+            "double_raw_type": "text",
+            "double_raw": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}',
+        }
+    ]
+
+
+@pytest.mark.asyncio
+async def test_base64_write_api_insert_upsert_update_decode_blobs(ds_write):
+    token = write_token(ds_write)
+    db = ds_write.get_database("data")
+    await db.execute_write(
+        "create table binary_api (id integer primary key, data blob, literal text)"
+    )
+
+    insert_response = await ds_write.client.post(
+        "/data/binary_api/-/insert",
+        json={
+            "row": {
+                "id": 1,
+                "data": BASE64_WRITE_API_VALUE,
+                "literal": {"$raw": BASE64_WRITE_API_VALUE},
+            }
+        },
+        headers=_headers(token),
+    )
+    assert insert_response.status_code == 201
+    assert insert_response.json()["rows"][0]["data"] == BASE64_WRITE_API_VALUE
+
+    upsert_response = await ds_write.client.post(
+        "/data/binary_api/-/upsert",
+        json={
+            "rows": [
+                {
+                    "id": 2,
+                    "data": BASE64_WRITE_API_VALUE,
+                    "literal": {"$raw": BASE64_WRITE_API_VALUE},
+                }
+            ]
+        },
+        headers=_headers(token),
+    )
+    assert upsert_response.status_code == 200
+    assert upsert_response.json() == {"ok": True}
+
+    update_response = await ds_write.client.post(
+        "/data/binary_api/1/-/update",
+        json={
+            "update": {
+                "data": {"$base64": True, "encoded": "/wAB"},
+                "literal": {"$raw": {"$raw": BASE64_WRITE_API_VALUE}},
+            },
+            "return": True,
+        },
+        headers=_headers(token),
+    )
+    assert update_response.status_code == 200
+    assert update_response.json()["row"]["data"] == {"$base64": True, "encoded": "/wAB"}
+
+    rows = (await db.execute("""
+            select
+              id,
+              typeof(data) as data_type,
+              hex(data) as data_hex,
+              typeof(literal) as literal_type,
+              literal
+            from binary_api
+            order by id
+            """)).dicts()
+    assert rows == [
+        {
+            "id": 1,
+            "data_type": "blob",
+            "data_hex": "FF0001",
+            "literal_type": "text",
+            "literal": '{"$raw": {"$base64": true, "encoded": "AAEC/f7/"}}',
+        },
+        {
+            "id": 2,
+            "data_type": "blob",
+            "data_hex": "000102FDFEFF",
+            "literal_type": "text",
+            "literal": BASE64_WRITE_API_LITERAL,
+        },
+    ]
+
+
 @pytest.mark.asyncio
 async def test_api_explorer_upsert_example_json(ds_write):
     response = await ds_write.client.get("/-/api", actor={"id": "root"})