max_post_body_bytes setting, enforced for reuest.post_body()

Closes #2823

datasette/__init__.py

@@ -2,7 +2,13 @@ from datasette.permissions import Permission  # noqa
 from datasette.version import __version_info__, __version__  # noqa
 from datasette.events import Event  # noqa
 from datasette.tokens import TokenHandler, TokenRestrictions  # noqa
-from datasette.utils.asgi import Forbidden, NotFound, Request, Response  # noqa
+from datasette.utils.asgi import (  # noqa
+    Forbidden,
+    NotFound,
+    PayloadTooLarge,
+    Request,
+    Response,
+)
 from datasette.utils import actor_matches_allow  # noqa
 from datasette.views import Context  # noqa
 from .hookspecs import hookimpl  # noqa

datasette/app.py

@@ -206,6 +206,11 @@ SETTINGS = (
         100,
         "Maximum rows that can be inserted at a time using the bulk insert API",
     ),
+    Setting(
+        "max_post_body_bytes",
+        2 * 1024 * 1024,
+        "Maximum size in bytes for a POST body read into memory, e.g. JSON API requests - set 0 to disable this limit",
+    ),
     Setting(
         "num_sql_threads",
         3,
@@ -2847,7 +2852,11 @@ class DatasetteRouter:
         if base_url != "/" and path.startswith(base_url):
             path = "/" + path[len(base_url) :]
             scope = dict(scope, route_path=path)
-        request = Request(scope, receive)
+        request = Request(
+            scope,
+            receive,
+            max_post_body_bytes=self.ds.setting("max_post_body_bytes"),
+        )
         # Populate request_messages if ds_messages cookie is present
         try:
             request._messages = self.ds.unsign(

datasette/utils/asgi.py

@@ -67,13 +67,25 @@ class BadRequest(Base400):
     status = 400
 
 
+class PayloadTooLarge(Base400):
+    status = 413
+
+
 SAMESITE_VALUES = ("strict", "lax", "none")
 
+# Bodies read fully into memory (post_body/post_vars/json) are capped at this
+# size unless the max_post_body_bytes setting says otherwise. Kept deliberately
+# far below multipart's DEFAULT_MAX_REQUEST_SIZE: that parser streams to disk,
+# while these bodies are held in RAM and json.loads() can multiply their
+# footprint several times over.
+DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024  # 2MB
+
 
 class Request:
-    def __init__(self, scope, receive):
+    def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES):
         self.scope = scope
         self.receive = receive
+        self.max_post_body_bytes = max_post_body_bytes
 
     def __repr__(self):
         return '<asgi.Request method="{}" url="{}">'.format(self.method, self.url)
@@ -141,15 +153,43 @@ class Request:
     def actor(self):
         return self.scope.get("actor", None)
 
-    async def post_body(self):
-        body = b""
+    async def post_body(self, max_bytes=None):
+        """
+        Read the request body fully into memory.
+
+        The body is capped at max_bytes - or self.max_post_body_bytes
+        (default 2MB, set from the max_post_body_bytes setting for requests
+        created by Datasette) if max_bytes is not provided. Pass max_bytes=0
+        to disable the limit. Raises PayloadTooLarge (HTTP 413) if exceeded -
+        oversized bodies are rejected as soon as the limit is passed, without
+        buffering the rest.
+        """
+        if max_bytes is None:
+            max_bytes = self.max_post_body_bytes
+        too_large = PayloadTooLarge(
+            "Request body exceeded maximum size of {} bytes".format(max_bytes)
+        )
+        if max_bytes:
+            # Reject early if the client declares an oversized body
+            try:
+                if int(self.headers.get("content-length", "")) > max_bytes:
+                    raise too_large
+            except ValueError:
+                # Missing or malformed - the streaming check below still applies
+                pass
+        chunks = []
+        received = 0
         more_body = True
         while more_body:
             message = await self.receive()
             assert message["type"] == "http.request", message
-            body += message.get("body", b"")
+            chunk = message.get("body", b"")
+            received += len(chunk)
+            if max_bytes and received > max_bytes:
+                raise too_large
+            chunks.append(chunk)
             more_body = message.get("more_body", False)
-        return body
+        return b"".join(chunks)
 
     async def post_vars(self):
         body = await self.post_body()

datasette/views/row.py

@@ -8,7 +8,7 @@ from dataclasses import dataclass, field
 import markupsafe
 import sqlite_utils
 
-from datasette.utils.asgi import NotFound, Forbidden, Response
+from datasette.utils.asgi import NotFound, Forbidden, PayloadTooLarge, Response
 from datasette.database import QueryInterrupted
 from datasette.events import UpdateRowEvent, DeleteRowEvent
 from datasette.resources import TableResource
@@ -798,6 +798,8 @@ class RowUpdateView(BaseView):
             data = await request.json()
         except json.JSONDecodeError as e:
             return _error(["Invalid JSON: {}".format(e)])
+        except PayloadTooLarge as e:
+            return _error([str(e)], 413)
 
         if not isinstance(data, dict):
             return _error(["JSON must be a dictionary"])

datasette/views/table.py

@@ -46,7 +46,14 @@ from datasette.utils import (
     WriteJsonValueError,
     sqlite3,
 )
-from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response
+from datasette.utils.asgi import (
+    BadRequest,
+    Forbidden,
+    NotFound,
+    PayloadTooLarge,
+    Request,
+    Response,
+)
 from datasette.filters import Filters
 import sqlite_utils
 from dataclasses import dataclass, field
@@ -1070,9 +1077,12 @@ class TableInsertView(BaseView):
 
         pks = await db.primary_keys(table_name)
 
-        rows, errors, extras = await self._validate_data(
-            request, db, table_name, pks, upsert
-        )
+        try:
+            rows, errors, extras = await self._validate_data(
+                request, db, table_name, pks, upsert
+            )
+        except PayloadTooLarge as e:
+            return _error([str(e)], 413)
         if errors:
             return _error(errors, 400)
         try:
@@ -1257,6 +1267,8 @@ class TableSetColumnTypeView(BaseView):
             data = await request.json()
         except json.JSONDecodeError as e:
             return _error(["Invalid JSON: {}".format(e)], 400)
+        except PayloadTooLarge as e:
+            return _error([str(e)], 413)
 
         if not isinstance(data, dict):
             return _error(["JSON must be a dictionary"], 400)
@@ -1375,6 +1387,8 @@ class TableDropView(BaseView):
             confirm = data.get("confirm")
         except json.JSONDecodeError:
             pass
+        except PayloadTooLarge as e:
+            return _error([str(e)], 413)
 
         if not confirm:
             return Response.json(

datasette/views/table_create_alter.py

@@ -26,7 +26,7 @@ from datasette.utils import (
     table_column_details,
     WriteJsonValueError,
 )
-from datasette.utils.asgi import NotFound, Response
+from datasette.utils.asgi import NotFound, PayloadTooLarge, Response
 from datasette.utils.sqlite import sqlite_hidden_table_names
 
 from .base import BaseView, _error
@@ -811,6 +811,8 @@ class TableCreateView(BaseView):
             data = await request.json()
         except json.JSONDecodeError as e:
             return _error(["Invalid JSON: {}".format(e)])
+        except PayloadTooLarge as e:
+            return _error([str(e)], 413)
 
         if not isinstance(data, dict):
             return _error(["JSON must be an object"])
@@ -1172,6 +1174,8 @@ class TableAlterView(BaseView):
             data = await request.json()
         except json.JSONDecodeError as e:
             return _error(["Invalid JSON: {}".format(e)], 400)
+        except PayloadTooLarge as e:
+            return _error([str(e)], 413)
 
         if not isinstance(data, dict):
             return _error(["JSON must be a dictionary"], 400)

docs/changelog.rst

@@ -15,6 +15,7 @@ Unreleased
 - Datasette's JSON APIs now consistently encode every ``BLOB`` value using the documented :ref:`binary value JSON format <binary_json_format>`, even when the bytes could be decoded as UTF-8 text.
 - The insert and edit row dialogs now provide a dedicated control for ``BLOB`` values. Existing binary values are shown by byte size, image values under 10MB are previewed as thumbnails, and replacements can be attached, dropped or pasted into the control.
 - The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata.
+- POST bodies that Datasette reads fully into memory - such as JSON submitted to the write API - are now capped by the new :ref:`setting_max_post_body_bytes` setting, defaulting to 2MB. Oversized requests are rejected with an HTTP 413 error as soon as the limit is exceeded, protecting smaller servers from memory exhaustion. File uploads are unaffected - ``request.form()`` streams those to disk and has its own separate limits.
 
 .. _v1_0_a35:
 

docs/cli-reference.rst

@@ -244,6 +244,9 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam
                                    custom query (default=1000)
       max_insert_rows              Maximum rows that can be inserted at a time using
                                    the bulk insert API (default=100)
+      max_post_body_bytes          Maximum size in bytes for a POST body read into
+                                   memory, e.g. JSON API requests - set 0 to disable
+                                   this limit (default=2097152)
       num_sql_threads              Number of threads in the thread pool for
                                    executing SQLite queries (default=3)
       sql_time_limit_ms            Time limit for a SQL query in milliseconds

docs/internals.rst

@@ -52,6 +52,9 @@ The request object is passed to various plugin hooks. It represents an incoming
 ``.actor`` - dictionary (str -> Any) or None
     The currently authenticated actor (see :ref:`actors <authentication_actor>`), or ``None`` if the request is unauthenticated.
 
+``.max_post_body_bytes`` - integer
+    The maximum number of bytes ``await request.post_body()`` will read into memory, or ``0`` for no limit. Set from the :ref:`setting_max_post_body_bytes` setting (default 2MB) for requests created by Datasette. Can be passed to the ``Request`` constructor as a keyword argument.
+
 The object also has the following awaitable methods:
 
 ``await request.form(files=False, ...)`` - FormData
@@ -109,9 +112,11 @@ The object also has the following awaitable methods:
 ``await request.json()`` - Any
     Returns the parsed JSON body of a request submitted by ``POST``.
 
-``await request.post_body()`` - bytes
+``await request.post_body(max_bytes=None)`` - bytes
     Returns the un-parsed body of a request submitted by ``POST`` - useful for things like incoming JSON data.
 
+    The body is read fully into memory, capped at ``request.max_post_body_bytes`` - which Datasette sets from the :ref:`setting_max_post_body_bytes` setting (default 2MB). Bodies that exceed the limit raise a ``datasette.PayloadTooLarge`` exception, which Datasette turns into an HTTP 413 error response. Pass ``max_bytes=`` to override the limit for a specific call, or ``max_bytes=0`` to disable it. ``request.post_vars()`` and ``request.json()`` read the body through this method, so the same limit applies to them.
+
 And a class method that can be used to create fake request objects for use in tests:
 
 ``fake(path_with_query_string, method="GET", scheme="http", url_vars=None)``

docs/settings.rst

@@ -125,6 +125,23 @@ You can increase or decrease this limit like so::
 
     datasette mydatabase.db --setting max_insert_rows 1000
 
+.. _setting_max_post_body_bytes:
+
+max_post_body_bytes
+~~~~~~~~~~~~~~~~~~~
+
+Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API <json_api_write>`. Requests with larger bodies are rejected with an HTTP 413 error. Defaults to 2,097,152 (2MB).
+
+This limit exists to protect against memory exhaustion: unlike file uploads handled by ``request.form()``, which stream to disk, these bodies are held entirely in memory and parsing them as JSON can multiply their memory footprint several times over.
+
+If you increase :ref:`setting_max_insert_rows` to support larger bulk inserts you may need to increase this limit as well::
+
+    datasette mydatabase.db --setting max_post_body_bytes 10485760
+
+Set it to 0 to disable the limit entirely::
+
+    datasette mydatabase.db --setting max_post_body_bytes 0
+
 .. _setting_num_sql_threads:
 
 num_sql_threads

tests/test_api.py

@@ -655,6 +655,7 @@ async def test_settings_json(ds_client):
         "facet_time_limit_ms": 200,
         "max_returned_rows": 100,
         "max_insert_rows": 100,
+        "max_post_body_bytes": 2 * 1024 * 1024,
         "sql_time_limit_ms": 200,
         "allow_download": True,
         "allow_signed_tokens": True,

tests/test_api_write.py

@@ -307,6 +307,35 @@ async def test_insert_rows(ds_write, return_rows):
         assert response.json()["rows"] == actual_rows
 
 
+@pytest.mark.asyncio
+async def test_insert_rows_post_body_too_large(tmp_path_factory):
+    db_path = str(tmp_path_factory.mktemp("dbs") / "data.db")
+    conn = sqlite3.connect(db_path)
+    conn.execute("create table docs (id integer primary key, title text)")
+    conn.close()
+    ds = Datasette([db_path], settings={"max_post_body_bytes": 100})
+    ds.root_enabled = True
+    token = write_token(ds)
+    response = await ds.client.post(
+        "/data/docs/-/insert",
+        json={"rows": [{"title": "x" * 200}]},
+        headers=_headers(token),
+    )
+    assert response.status_code == 413
+    assert response.json() == {
+        "ok": False,
+        "errors": ["Request body exceeded maximum size of 100 bytes"],
+    }
+    # A small body should still work
+    response2 = await ds.client.post(
+        "/data/docs/-/insert",
+        json={"row": {"title": "hi"}},
+        headers=_headers(token),
+    )
+    assert response2.status_code == 201
+    ds.close()
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "path,input,special_case,expected_status,expected_errors",

tests/test_internals_request.py

@@ -1,8 +1,38 @@
-from datasette.utils.asgi import Request
+from datasette.utils.asgi import PayloadTooLarge, Request
 import json
 import pytest
 
 
+def _post_scope(headers=None):
+    return {
+        "http_version": "1.1",
+        "method": "POST",
+        "path": "/",
+        "raw_path": b"/",
+        "query_string": b"",
+        "scheme": "http",
+        "type": "http",
+        "headers": headers or [[b"content-type", b"application/json"]],
+    }
+
+
+def _receive_chunks(chunks):
+    messages = [
+        {
+            "type": "http.request",
+            "body": chunk,
+            "more_body": i < len(chunks) - 1,
+        }
+        for i, chunk in enumerate(chunks)
+    ]
+    messages.reverse()
+
+    async def receive():
+        return messages.pop()
+
+    return receive
+
+
 @pytest.mark.asyncio
 async def test_request_post_vars():
     scope = {
@@ -106,6 +136,70 @@ async def test_request_json_invalid():
         await request.json()
 
 
+@pytest.mark.asyncio
+async def test_request_post_body_multiple_chunks():
+    request = Request(_post_scope(), _receive_chunks([b"hello ", b"world"]))
+    assert await request.post_body() == b"hello world"
+
+
+@pytest.mark.asyncio
+async def test_request_post_body_content_length_too_large():
+    # Should reject based on content-length without reading the body
+    async def receive():
+        raise AssertionError("receive() should not be called")
+
+    scope = _post_scope(
+        headers=[
+            [b"content-type", b"application/json"],
+            [b"content-length", b"101"],
+        ]
+    )
+    request = Request(scope, receive)
+    with pytest.raises(PayloadTooLarge):
+        await request.post_body(max_bytes=100)
+
+
+@pytest.mark.asyncio
+async def test_request_post_body_streaming_too_large():
+    # No content-length header - limit enforced as chunks arrive
+    chunks = [b"a" * 60, b"b" * 60, b"c" * 60]
+    request = Request(_post_scope(), _receive_chunks(chunks))
+    with pytest.raises(PayloadTooLarge):
+        await request.post_body(max_bytes=100)
+
+
+@pytest.mark.asyncio
+async def test_request_post_body_limit_from_constructor():
+    request = Request(
+        _post_scope(), _receive_chunks([b"too much data"]), max_post_body_bytes=5
+    )
+    with pytest.raises(PayloadTooLarge):
+        await request.post_body()
+
+
+@pytest.mark.asyncio
+async def test_request_post_body_limit_disabled():
+    body = b"a" * (3 * 1024 * 1024)
+    request = Request(_post_scope(), _receive_chunks([body]), max_post_body_bytes=0)
+    assert await request.post_body() == body
+
+
+@pytest.mark.asyncio
+async def test_request_post_body_default_limit():
+    # Bodies over 2MB are rejected by default
+    request = Request(_post_scope(), _receive_chunks([b"a" * (2 * 1024 * 1024 + 1)]))
+    with pytest.raises(PayloadTooLarge):
+        await request.post_body()
+
+
+@pytest.mark.asyncio
+async def test_request_json_too_large():
+    body = json.dumps({"rows": ["x" * 100]}).encode("utf-8")
+    request = Request(_post_scope(), _receive_chunks([body]), max_post_body_bytes=50)
+    with pytest.raises(PayloadTooLarge):
+        await request.json()
+
+
 def test_request_args():
     request = Request.fake("/foo?multi=1&multi=2&single=3")
     assert "1" == request.args.get("multi")