Merge remote-tracking branch 'origin/main' into claude/json-api-docs-1-0-review-a3e83u

# Conflicts:
#	datasette/__init__.py
#	tests/test_api_write.py

datasette/utils/__init__.py

@@ -1,4 +1,5 @@
 import asyncio
+import binascii
 from contextlib import contextmanager
 import aiofiles
 import click
@@ -236,10 +237,8 @@ class CustomJSONEncoder(json.JSONEncoder):
     - ``sqlite3.Row`` becomes a tuple
     - ``sqlite3.Cursor`` becomes a list
 
-    If a binary blob can be decoded as UTF-8, the encoder returns it as text.
-
-    If it can't (for example, images), it is encoded as an object, with the actual
-    data base64-encoded, like so: ::
+    Binary blobs are encoded as an object, with the actual data base64-encoded,
+    like so: ::
 
         {
             "$base64": True,
@@ -255,17 +254,42 @@ class CustomJSONEncoder(json.JSONEncoder):
         if isinstance(obj, sqlite3.Cursor):
             return list(obj)
         if isinstance(obj, bytes):
-            # Does it encode to utf8?
-            try:
-                return obj.decode("utf8")
-            except UnicodeDecodeError:
-                return {
-                    "$base64": True,
-                    "encoded": base64.b64encode(obj).decode("latin1"),
-                }
+            return {
+                "$base64": True,
+                "encoded": base64.b64encode(obj).decode("latin1"),
+            }
         return json.JSONEncoder.default(self, obj)
 
 
+class WriteJsonValueError(ValueError):
+    pass
+
+
+def decode_write_json_cell(value):
+    if not isinstance(value, dict):
+        return value
+    keys = set(value.keys())
+    if keys == {"$raw"}:
+        return value["$raw"]
+    if keys == {"$base64", "encoded"} and value.get("$base64") is True:
+        encoded = value["encoded"]
+        if not isinstance(encoded, str):
+            raise WriteJsonValueError("$base64 encoded value must be a string")
+        try:
+            return base64.b64decode(encoded, validate=True)
+        except binascii.Error as ex:
+            raise WriteJsonValueError("Invalid $base64 encoded value") from ex
+    return value
+
+
+def decode_write_json_row(row):
+    return {key: decode_write_json_cell(value) for key, value in row.items()}
+
+
+def decode_write_json_rows(rows):
+    return [decode_write_json_row(row) for row in rows]
+
+
 @contextmanager
 def sqlite_timelimit(conn, ms):
     deadline = time.perf_counter() + (ms / 1000)

datasette/utils/asgi.py

@@ -67,13 +67,25 @@ class BadRequest(Base400):
     status = 400
 
 
+class PayloadTooLarge(Base400):
+    status = 413
+
+
 SAMESITE_VALUES = ("strict", "lax", "none")
 
+# Bodies read fully into memory (post_body/post_vars/json) are capped at this
+# size unless the max_post_body_bytes setting says otherwise. Kept deliberately
+# far below multipart's DEFAULT_MAX_REQUEST_SIZE: that parser streams to disk,
+# while these bodies are held in RAM and json.loads() can multiply their
+# footprint several times over.
+DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024  # 2MB
+
 
 class Request:
-    def __init__(self, scope, receive):
+    def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES):
         self.scope = scope
         self.receive = receive
+        self.max_post_body_bytes = max_post_body_bytes
 
     def __repr__(self):
         return '<asgi.Request method="{}" url="{}">'.format(self.method, self.url)
@@ -141,15 +153,43 @@ class Request:
     def actor(self):
         return self.scope.get("actor", None)
 
-    async def post_body(self):
-        body = b""
+    async def post_body(self, max_bytes=None):
+        """
+        Read the request body fully into memory.
+
+        The body is capped at max_bytes - or self.max_post_body_bytes
+        (default 2MB, set from the max_post_body_bytes setting for requests
+        created by Datasette) if max_bytes is not provided. Pass max_bytes=0
+        to disable the limit. Raises PayloadTooLarge (HTTP 413) if exceeded -
+        oversized bodies are rejected as soon as the limit is passed, without
+        buffering the rest.
+        """
+        if max_bytes is None:
+            max_bytes = self.max_post_body_bytes
+        too_large = PayloadTooLarge(
+            "Request body exceeded maximum size of {} bytes".format(max_bytes)
+        )
+        if max_bytes:
+            # Reject early if the client declares an oversized body
+            try:
+                if int(self.headers.get("content-length", "")) > max_bytes:
+                    raise too_large
+            except ValueError:
+                # Missing or malformed - the streaming check below still applies
+                pass
+        chunks = []
+        received = 0
         more_body = True
         while more_body:
             message = await self.receive()
             assert message["type"] == "http.request", message
-            body += message.get("body", b"")
+            chunk = message.get("body", b"")
+            received += len(chunk)
+            if max_bytes and received > max_bytes:
+                raise too_large
+            chunks.append(chunk)
             more_body = message.get("more_body", False)
-        return body
+        return b"".join(chunks)
 
     async def post_vars(self):
         body = await self.post_body()

datasette/utils/sql_analysis.py

@@ -488,17 +488,17 @@ def analyze_sql_tables(
         and key.operation in {"create", "alter", "drop"}
         for key in operations
     )
-    dropped_tables = {
+    dropped_tables_and_views = {
         (key.database, key.table)
         for key in operations
-        if key.operation == "drop" and key.target_type == "table"
+        if key.operation == "drop" and key.target_type in {"table", "view"}
     }
 
     def key_is_drop_table_delete(key: OperationKey) -> bool:
         return (
             key.operation == "delete"
             and key.target_type == "table"
-            and (key.database, key.table) in dropped_tables
+            and (key.database, key.table) in dropped_tables_and_views
         )
 
     has_user_table_access_in_schema_operation = any(