diff --git a/datasette/__init__.py b/datasette/__init__.py index eb18e59e..4c96a148 100644 --- a/datasette/__init__.py +++ b/datasette/__init__.py @@ -2,7 +2,13 @@ from datasette.permissions import Permission # noqa from datasette.version import __version_info__, __version__ # noqa from datasette.events import Event # noqa from datasette.tokens import TokenHandler, TokenRestrictions # noqa -from datasette.utils.asgi import Forbidden, NotFound, Request, Response # noqa +from datasette.utils.asgi import ( # noqa + Forbidden, + NotFound, + PayloadTooLarge, + Request, + Response, +) from datasette.utils import actor_matches_allow # noqa from datasette.views import Context # noqa from .hookspecs import hookimpl # noqa diff --git a/datasette/app.py b/datasette/app.py index 9c9b7de4..2bcd50f6 100644 --- a/datasette/app.py +++ b/datasette/app.py @@ -206,6 +206,11 @@ SETTINGS = ( 100, "Maximum rows that can be inserted at a time using the bulk insert API", ), + Setting( + "max_post_body_bytes", + 2 * 1024 * 1024, + "Maximum size in bytes for a POST body read into memory, e.g. JSON API requests - set 0 to disable this limit", + ), Setting( "num_sql_threads", 3, @@ -2847,7 +2852,11 @@ class DatasetteRouter: if base_url != "/" and path.startswith(base_url): path = "/" + path[len(base_url) :] scope = dict(scope, route_path=path) - request = Request(scope, receive) + request = Request( + scope, + receive, + max_post_body_bytes=self.ds.setting("max_post_body_bytes"), + ) # Populate request_messages if ds_messages cookie is present try: request._messages = self.ds.unsign( diff --git a/datasette/utils/asgi.py b/datasette/utils/asgi.py index e1631b10..7777308f 100644 --- a/datasette/utils/asgi.py +++ b/datasette/utils/asgi.py @@ -67,13 +67,25 @@ class BadRequest(Base400): status = 400 +class PayloadTooLarge(Base400): + status = 413 + + SAMESITE_VALUES = ("strict", "lax", "none") +# Bodies read fully into memory (post_body/post_vars/json) are capped at this +# size unless the max_post_body_bytes setting says otherwise. Kept deliberately +# far below multipart's DEFAULT_MAX_REQUEST_SIZE: that parser streams to disk, +# while these bodies are held in RAM and json.loads() can multiply their +# footprint several times over. +DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024 # 2MB + class Request: - def __init__(self, scope, receive): + def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES): self.scope = scope self.receive = receive + self.max_post_body_bytes = max_post_body_bytes def __repr__(self): return ''.format(self.method, self.url) @@ -141,15 +153,43 @@ class Request: def actor(self): return self.scope.get("actor", None) - async def post_body(self): - body = b"" + async def post_body(self, max_bytes=None): + """ + Read the request body fully into memory. + + The body is capped at max_bytes - or self.max_post_body_bytes + (default 2MB, set from the max_post_body_bytes setting for requests + created by Datasette) if max_bytes is not provided. Pass max_bytes=0 + to disable the limit. Raises PayloadTooLarge (HTTP 413) if exceeded - + oversized bodies are rejected as soon as the limit is passed, without + buffering the rest. + """ + if max_bytes is None: + max_bytes = self.max_post_body_bytes + too_large = PayloadTooLarge( + "Request body exceeded maximum size of {} bytes".format(max_bytes) + ) + if max_bytes: + # Reject early if the client declares an oversized body + try: + if int(self.headers.get("content-length", "")) > max_bytes: + raise too_large + except ValueError: + # Missing or malformed - the streaming check below still applies + pass + chunks = [] + received = 0 more_body = True while more_body: message = await self.receive() assert message["type"] == "http.request", message - body += message.get("body", b"") + chunk = message.get("body", b"") + received += len(chunk) + if max_bytes and received > max_bytes: + raise too_large + chunks.append(chunk) more_body = message.get("more_body", False) - return body + return b"".join(chunks) async def post_vars(self): body = await self.post_body() diff --git a/datasette/views/row.py b/datasette/views/row.py index f84d3a09..5cf2c541 100644 --- a/datasette/views/row.py +++ b/datasette/views/row.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field import markupsafe import sqlite_utils -from datasette.utils.asgi import NotFound, Forbidden, Response +from datasette.utils.asgi import NotFound, Forbidden, PayloadTooLarge, Response from datasette.database import QueryInterrupted from datasette.events import UpdateRowEvent, DeleteRowEvent from datasette.resources import TableResource @@ -798,6 +798,8 @@ class RowUpdateView(BaseView): data = await request.json() except json.JSONDecodeError as e: return _error(["Invalid JSON: {}".format(e)]) + except PayloadTooLarge as e: + return _error([str(e)], 413) if not isinstance(data, dict): return _error(["JSON must be a dictionary"]) diff --git a/datasette/views/table.py b/datasette/views/table.py index 73652f05..2e698ac3 100644 --- a/datasette/views/table.py +++ b/datasette/views/table.py @@ -46,7 +46,14 @@ from datasette.utils import ( WriteJsonValueError, sqlite3, ) -from datasette.utils.asgi import BadRequest, Forbidden, NotFound, Request, Response +from datasette.utils.asgi import ( + BadRequest, + Forbidden, + NotFound, + PayloadTooLarge, + Request, + Response, +) from datasette.filters import Filters import sqlite_utils from dataclasses import dataclass, field @@ -1070,9 +1077,12 @@ class TableInsertView(BaseView): pks = await db.primary_keys(table_name) - rows, errors, extras = await self._validate_data( - request, db, table_name, pks, upsert - ) + try: + rows, errors, extras = await self._validate_data( + request, db, table_name, pks, upsert + ) + except PayloadTooLarge as e: + return _error([str(e)], 413) if errors: return _error(errors, 400) try: @@ -1257,6 +1267,8 @@ class TableSetColumnTypeView(BaseView): data = await request.json() except json.JSONDecodeError as e: return _error(["Invalid JSON: {}".format(e)], 400) + except PayloadTooLarge as e: + return _error([str(e)], 413) if not isinstance(data, dict): return _error(["JSON must be a dictionary"], 400) @@ -1375,6 +1387,8 @@ class TableDropView(BaseView): confirm = data.get("confirm") except json.JSONDecodeError: pass + except PayloadTooLarge as e: + return _error([str(e)], 413) if not confirm: return Response.json( diff --git a/datasette/views/table_create_alter.py b/datasette/views/table_create_alter.py index 12b8e9f9..d0384184 100644 --- a/datasette/views/table_create_alter.py +++ b/datasette/views/table_create_alter.py @@ -26,7 +26,7 @@ from datasette.utils import ( table_column_details, WriteJsonValueError, ) -from datasette.utils.asgi import NotFound, Response +from datasette.utils.asgi import NotFound, PayloadTooLarge, Response from datasette.utils.sqlite import sqlite_hidden_table_names from .base import BaseView, _error @@ -811,6 +811,8 @@ class TableCreateView(BaseView): data = await request.json() except json.JSONDecodeError as e: return _error(["Invalid JSON: {}".format(e)]) + except PayloadTooLarge as e: + return _error([str(e)], 413) if not isinstance(data, dict): return _error(["JSON must be an object"]) @@ -1172,6 +1174,8 @@ class TableAlterView(BaseView): data = await request.json() except json.JSONDecodeError as e: return _error(["Invalid JSON: {}".format(e)], 400) + except PayloadTooLarge as e: + return _error([str(e)], 413) if not isinstance(data, dict): return _error(["JSON must be a dictionary"], 400) diff --git a/docs/changelog.rst b/docs/changelog.rst index 4e8f33d8..4541981f 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -15,6 +15,7 @@ Unreleased - Datasette's JSON APIs now consistently encode every ``BLOB`` value using the documented :ref:`binary value JSON format `, even when the bytes could be decoded as UTF-8 text. - The insert and edit row dialogs now provide a dedicated control for ``BLOB`` values. Existing binary values are shown by byte size, image values under 10MB are previewed as thumbnails, and replacements can be attached, dropped or pasted into the control. - The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata. +- POST bodies that Datasette reads fully into memory - such as JSON submitted to the write API - are now capped by the new :ref:`setting_max_post_body_bytes` setting, defaulting to 2MB. Oversized requests are rejected with an HTTP 413 error as soon as the limit is exceeded, protecting smaller servers from memory exhaustion. File uploads are unaffected - ``request.form()`` streams those to disk and has its own separate limits. .. _v1_0_a35: diff --git a/docs/cli-reference.rst b/docs/cli-reference.rst index 7ca88c4e..2302f742 100644 --- a/docs/cli-reference.rst +++ b/docs/cli-reference.rst @@ -244,6 +244,9 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam custom query (default=1000) max_insert_rows Maximum rows that can be inserted at a time using the bulk insert API (default=100) + max_post_body_bytes Maximum size in bytes for a POST body read into + memory, e.g. JSON API requests - set 0 to disable + this limit (default=2097152) num_sql_threads Number of threads in the thread pool for executing SQLite queries (default=3) sql_time_limit_ms Time limit for a SQL query in milliseconds diff --git a/docs/internals.rst b/docs/internals.rst index 9f75640f..e5e7aca5 100644 --- a/docs/internals.rst +++ b/docs/internals.rst @@ -52,6 +52,9 @@ The request object is passed to various plugin hooks. It represents an incoming ``.actor`` - dictionary (str -> Any) or None The currently authenticated actor (see :ref:`actors `), or ``None`` if the request is unauthenticated. +``.max_post_body_bytes`` - integer + The maximum number of bytes ``await request.post_body()`` will read into memory, or ``0`` for no limit. Set from the :ref:`setting_max_post_body_bytes` setting (default 2MB) for requests created by Datasette. Can be passed to the ``Request`` constructor as a keyword argument. + The object also has the following awaitable methods: ``await request.form(files=False, ...)`` - FormData @@ -109,9 +112,11 @@ The object also has the following awaitable methods: ``await request.json()`` - Any Returns the parsed JSON body of a request submitted by ``POST``. -``await request.post_body()`` - bytes +``await request.post_body(max_bytes=None)`` - bytes Returns the un-parsed body of a request submitted by ``POST`` - useful for things like incoming JSON data. + The body is read fully into memory, capped at ``request.max_post_body_bytes`` - which Datasette sets from the :ref:`setting_max_post_body_bytes` setting (default 2MB). Bodies that exceed the limit raise a ``datasette.PayloadTooLarge`` exception, which Datasette turns into an HTTP 413 error response. Pass ``max_bytes=`` to override the limit for a specific call, or ``max_bytes=0`` to disable it. ``request.post_vars()`` and ``request.json()`` read the body through this method, so the same limit applies to them. + And a class method that can be used to create fake request objects for use in tests: ``fake(path_with_query_string, method="GET", scheme="http", url_vars=None)`` diff --git a/docs/settings.rst b/docs/settings.rst index 5cd49113..9c114e4a 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -125,6 +125,23 @@ You can increase or decrease this limit like so:: datasette mydatabase.db --setting max_insert_rows 1000 +.. _setting_max_post_body_bytes: + +max_post_body_bytes +~~~~~~~~~~~~~~~~~~~ + +Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API `. Requests with larger bodies are rejected with an HTTP 413 error. Defaults to 2,097,152 (2MB). + +This limit exists to protect against memory exhaustion: unlike file uploads handled by ``request.form()``, which stream to disk, these bodies are held entirely in memory and parsing them as JSON can multiply their memory footprint several times over. + +If you increase :ref:`setting_max_insert_rows` to support larger bulk inserts you may need to increase this limit as well:: + + datasette mydatabase.db --setting max_post_body_bytes 10485760 + +Set it to 0 to disable the limit entirely:: + + datasette mydatabase.db --setting max_post_body_bytes 0 + .. _setting_num_sql_threads: num_sql_threads diff --git a/tests/test_api.py b/tests/test_api.py index d99c3341..17826989 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -655,6 +655,7 @@ async def test_settings_json(ds_client): "facet_time_limit_ms": 200, "max_returned_rows": 100, "max_insert_rows": 100, + "max_post_body_bytes": 2 * 1024 * 1024, "sql_time_limit_ms": 200, "allow_download": True, "allow_signed_tokens": True, diff --git a/tests/test_api_write.py b/tests/test_api_write.py index 1a4a966b..1154176f 100644 --- a/tests/test_api_write.py +++ b/tests/test_api_write.py @@ -307,6 +307,35 @@ async def test_insert_rows(ds_write, return_rows): assert response.json()["rows"] == actual_rows +@pytest.mark.asyncio +async def test_insert_rows_post_body_too_large(tmp_path_factory): + db_path = str(tmp_path_factory.mktemp("dbs") / "data.db") + conn = sqlite3.connect(db_path) + conn.execute("create table docs (id integer primary key, title text)") + conn.close() + ds = Datasette([db_path], settings={"max_post_body_bytes": 100}) + ds.root_enabled = True + token = write_token(ds) + response = await ds.client.post( + "/data/docs/-/insert", + json={"rows": [{"title": "x" * 200}]}, + headers=_headers(token), + ) + assert response.status_code == 413 + assert response.json() == { + "ok": False, + "errors": ["Request body exceeded maximum size of 100 bytes"], + } + # A small body should still work + response2 = await ds.client.post( + "/data/docs/-/insert", + json={"row": {"title": "hi"}}, + headers=_headers(token), + ) + assert response2.status_code == 201 + ds.close() + + @pytest.mark.asyncio @pytest.mark.parametrize( "path,input,special_case,expected_status,expected_errors", diff --git a/tests/test_internals_request.py b/tests/test_internals_request.py index 9c448186..6d2dc70a 100644 --- a/tests/test_internals_request.py +++ b/tests/test_internals_request.py @@ -1,8 +1,38 @@ -from datasette.utils.asgi import Request +from datasette.utils.asgi import PayloadTooLarge, Request import json import pytest +def _post_scope(headers=None): + return { + "http_version": "1.1", + "method": "POST", + "path": "/", + "raw_path": b"/", + "query_string": b"", + "scheme": "http", + "type": "http", + "headers": headers or [[b"content-type", b"application/json"]], + } + + +def _receive_chunks(chunks): + messages = [ + { + "type": "http.request", + "body": chunk, + "more_body": i < len(chunks) - 1, + } + for i, chunk in enumerate(chunks) + ] + messages.reverse() + + async def receive(): + return messages.pop() + + return receive + + @pytest.mark.asyncio async def test_request_post_vars(): scope = { @@ -106,6 +136,70 @@ async def test_request_json_invalid(): await request.json() +@pytest.mark.asyncio +async def test_request_post_body_multiple_chunks(): + request = Request(_post_scope(), _receive_chunks([b"hello ", b"world"])) + assert await request.post_body() == b"hello world" + + +@pytest.mark.asyncio +async def test_request_post_body_content_length_too_large(): + # Should reject based on content-length without reading the body + async def receive(): + raise AssertionError("receive() should not be called") + + scope = _post_scope( + headers=[ + [b"content-type", b"application/json"], + [b"content-length", b"101"], + ] + ) + request = Request(scope, receive) + with pytest.raises(PayloadTooLarge): + await request.post_body(max_bytes=100) + + +@pytest.mark.asyncio +async def test_request_post_body_streaming_too_large(): + # No content-length header - limit enforced as chunks arrive + chunks = [b"a" * 60, b"b" * 60, b"c" * 60] + request = Request(_post_scope(), _receive_chunks(chunks)) + with pytest.raises(PayloadTooLarge): + await request.post_body(max_bytes=100) + + +@pytest.mark.asyncio +async def test_request_post_body_limit_from_constructor(): + request = Request( + _post_scope(), _receive_chunks([b"too much data"]), max_post_body_bytes=5 + ) + with pytest.raises(PayloadTooLarge): + await request.post_body() + + +@pytest.mark.asyncio +async def test_request_post_body_limit_disabled(): + body = b"a" * (3 * 1024 * 1024) + request = Request(_post_scope(), _receive_chunks([body]), max_post_body_bytes=0) + assert await request.post_body() == body + + +@pytest.mark.asyncio +async def test_request_post_body_default_limit(): + # Bodies over 2MB are rejected by default + request = Request(_post_scope(), _receive_chunks([b"a" * (2 * 1024 * 1024 + 1)])) + with pytest.raises(PayloadTooLarge): + await request.post_body() + + +@pytest.mark.asyncio +async def test_request_json_too_large(): + body = json.dumps({"rows": ["x" * 100]}).encode("utf-8") + request = Request(_post_scope(), _receive_chunks([body]), max_post_body_bytes=50) + with pytest.raises(PayloadTooLarge): + await request.json() + + def test_request_args(): request = Request.fake("/foo?multi=1&multi=2&single=3") assert "1" == request.args.get("multi")