Compare commits

...

2 commits

Author SHA1 Message Date
Simon Willison
ae1145475a No need to describe pytest-httpx any more 2026-08-13 17:11:39 -07:00
Simon Willison
f712559886 Port to httpx2, refs #2879 2026-08-13 16:22:30 -07:00
10 changed files with 62 additions and 124 deletions

View file

@ -28,7 +28,7 @@ import urllib.parse
from concurrent import futures from concurrent import futures
from pathlib import Path from pathlib import Path
import httpx import httpx2
from itsdangerous import BadSignature, URLSafeSerializer from itsdangerous import BadSignature, URLSafeSerializer
from jinja2 import ( from jinja2 import (
ChoiceLoader, ChoiceLoader,
@ -3215,14 +3215,14 @@ class DatasetteClient:
with _DatasetteClientContext(): with _DatasetteClientContext():
if skip_permission_checks: if skip_permission_checks:
with SkipPermissions(): with SkipPermissions():
async with httpx.AsyncClient( async with httpx2.AsyncClient(
transport=httpx.ASGITransport(app=self.app), transport=httpx2.ASGITransport(app=self.app),
cookies=kwargs.pop("cookies", None), cookies=kwargs.pop("cookies", None),
) as client: ) as client:
return await getattr(client, method)(self._fix(path), **kwargs) return await getattr(client, method)(self._fix(path), **kwargs)
else: else:
async with httpx.AsyncClient( async with httpx2.AsyncClient(
transport=httpx.ASGITransport(app=self.app), transport=httpx2.ASGITransport(app=self.app),
cookies=kwargs.pop("cookies", None), cookies=kwargs.pop("cookies", None),
) as client: ) as client:
return await getattr(client, method)(self._fix(path), **kwargs) return await getattr(client, method)(self._fix(path), **kwargs)
@ -3269,10 +3269,10 @@ class DatasetteClient:
method: HTTP method (e.g., "GET", "POST", "PUT") method: HTTP method (e.g., "GET", "POST", "PUT")
path: The path to request path: The path to request
skip_permission_checks: If True, bypass all permission checks for this request skip_permission_checks: If True, bypass all permission checks for this request
**kwargs: Additional arguments to pass to httpx **kwargs: Additional arguments to pass to httpx2
Returns: Returns:
httpx.Response: The response from the request httpx2.Response: The response from the request
""" """
from datasette.permissions import SkipPermissions from datasette.permissions import SkipPermissions
@ -3281,16 +3281,16 @@ class DatasetteClient:
with _DatasetteClientContext(): with _DatasetteClientContext():
if skip_permission_checks: if skip_permission_checks:
with SkipPermissions(): with SkipPermissions():
async with httpx.AsyncClient( async with httpx2.AsyncClient(
transport=httpx.ASGITransport(app=self.app), transport=httpx2.ASGITransport(app=self.app),
cookies=kwargs.pop("cookies", None), cookies=kwargs.pop("cookies", None),
) as client: ) as client:
return await client.request( return await client.request(
method, self._fix(path, avoid_path_rewrites), **kwargs method, self._fix(path, avoid_path_rewrites), **kwargs
) )
else: else:
async with httpx.AsyncClient( async with httpx2.AsyncClient(
transport=httpx.ASGITransport(app=self.app), transport=httpx2.ASGITransport(app=self.app),
cookies=kwargs.pop("cookies", None), cookies=kwargs.pop("cookies", None),
) as client: ) as client:
return await client.request( return await client.request(

View file

@ -4,7 +4,7 @@ from urllib.parse import urlencode
from asgiref.sync import async_to_sync from asgiref.sync import async_to_sync
# These wrapper classes pre-date the introduction of # These wrapper classes pre-date the introduction of
# datasette.client and httpx to Datasette. They could # datasette.client and HTTPX2 to Datasette. They could
# be removed if the Datasette tests are modified to # be removed if the Datasette tests are modified to
# call datasette.client directly. # call datasette.client directly.

View file

@ -4,6 +4,13 @@
Changelog Changelog
========= =========
.. _v1_0_unreleased:
Unreleased
----------
- Datasette now uses `HTTPX2 <https://httpx2.pydantic.dev/>`__ in place of HTTPX. (:issue:`2879`)
.. _v1_0_a38: .. _v1_0_a38:
1.0a38 (2026-08-06) 1.0a38 (2026-08-06)
@ -1429,7 +1436,7 @@ Other changes
- The request object now provides a ``request.full_path`` property, which returns the path including any query string. (:issue:`1184`) - The request object now provides a ``request.full_path`` property, which returns the path including any query string. (:issue:`1184`)
- Better error message for disallowed ``PRAGMA`` clauses in SQL queries. (:issue:`1185`) - Better error message for disallowed ``PRAGMA`` clauses in SQL queries. (:issue:`1185`)
- ``datasette publish heroku`` now deploys using ``python-3.8.7``. - ``datasette publish heroku`` now deploys using ``python-3.8.7``.
- New plugin testing documentation on :ref:`testing_plugins_pytest_httpx`. (:issue:`1198`) - New plugin testing documentation for mocking outbound HTTP calls. (:issue:`1198`)
- All ``?_*`` query string parameters passed to the table page are now persisted in hidden form fields, so parameters such as ``?_size=10`` will be correctly passed to the next page when query filters are changed. (:issue:`1194`) - All ``?_*`` query string parameters passed to the table page are now persisted in hidden form fields, so parameters such as ``?_size=10`` will be correctly passed to the next page when query filters are changed. (:issue:`1194`)
- Fixed a bug loading a database file called ``test-database (1).sqlite``. (:issue:`1181`) - Fixed a bug loading a database file called ``test-database (1).sqlite``. (:issue:`1181`)

View file

@ -1594,32 +1594,32 @@ datasette.client
Plugins can make internal simulated HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins, while avoiding the overhead of making an external HTTP call to access those APIs. Plugins can make internal simulated HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins, while avoiding the overhead of making an external HTTP call to access those APIs.
The ``datasette.client`` object is a wrapper around the `HTTPX Python library <https://www.python-httpx.org/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__. The ``datasette.client`` object is a wrapper around the `HTTPX2 Python library <https://httpx2.pydantic.dev/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
It offers the following methods: It offers the following methods:
``await datasette.client.get(path, **kwargs)`` - returns HTTPX Response ``await datasette.client.get(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal GET request against that path. Execute an internal GET request against that path.
``await datasette.client.post(path, **kwargs)`` - returns HTTPX Response ``await datasette.client.post(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal POST request. Use ``data={"name": "value"}`` to pass form parameters. Execute an internal POST request. Use ``data={"name": "value"}`` to pass form parameters.
``await datasette.client.options(path, **kwargs)`` - returns HTTPX Response ``await datasette.client.options(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal OPTIONS request. Execute an internal OPTIONS request.
``await datasette.client.head(path, **kwargs)`` - returns HTTPX Response ``await datasette.client.head(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal HEAD request. Execute an internal HEAD request.
``await datasette.client.put(path, **kwargs)`` - returns HTTPX Response ``await datasette.client.put(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal PUT request. Execute an internal PUT request.
``await datasette.client.patch(path, **kwargs)`` - returns HTTPX Response ``await datasette.client.patch(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal PATCH request. Execute an internal PATCH request.
``await datasette.client.delete(path, **kwargs)`` - returns HTTPX Response ``await datasette.client.delete(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal DELETE request. Execute an internal DELETE request.
``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX Response ``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX2 Response
Execute an internal request with the given HTTP method against that path. Execute an internal request with the given HTTP method against that path.
These methods can be used with :ref:`internals_datasette_urls` - for example: These methods can be used with :ref:`internals_datasette_urls` - for example:
@ -1636,7 +1636,7 @@ These methods can be used with :ref:`internals_datasette_urls` - for example:
``datasette.client`` methods automatically take the current :ref:`setting_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path. ``datasette.client`` methods automatically take the current :ref:`setting_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path.
For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation <https://www.python-httpx.org/async/>`__. For documentation on available ``**kwargs`` options and the shape of the HTTPX2 Response object refer to the `HTTPX2 Async documentation <https://httpx2.pydantic.dev/async/>`__.
.. _internals_datasette_client_actor: .. _internals_datasette_client_actor:
@ -2623,12 +2623,12 @@ This example uses trace to record the start, end and duration of any HTTP GET re
.. code-block:: python .. code-block:: python
from datasette.tracer import trace from datasette.tracer import trace
import httpx import httpx2
async def fetch_url(url): async def fetch_url(url):
with trace("fetch-url", url=url): with trace("fetch-url", url=url):
async with httpx.AsyncClient() as client: async with httpx2.AsyncClient() as client:
return await client.get(url) return await client.get(url)
.. _internals_tracer_trace_child_tasks: .. _internals_tracer_trace_child_tasks:

View file

@ -25,7 +25,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl
) )
This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX <https://www.python-httpx.org/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance. This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX2 <https://httpx2.pydantic.dev/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
This test also uses the `pytest-asyncio <https://pypi.org/project/pytest-asyncio/>`__ package to add support for ``async def`` test functions running under pytest. This test also uses the `pytest-asyncio <https://pypi.org/project/pytest-asyncio/>`__ package to add support for ``async def`` test functions running under pytest.
@ -154,7 +154,7 @@ If you need to opt out of this behavior, add the following to your ``pytest.ini`
Using datasette.client in tests Using datasette.client in tests
------------------------------- -------------------------------
The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client <https://www.python-httpx.org/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test. The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX2 async client <https://httpx2.pydantic.dev/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
A simple test looks like this: A simple test looks like this:
@ -271,75 +271,6 @@ If you want to create that test database repeatedly for every individual test fu
# This fixture will be executed repeatedly for every test # This fixture will be executed repeatedly for every test
... ...
.. _testing_plugins_pytest_httpx:
Testing outbound HTTP calls with pytest-httpx
---------------------------------------------
If your plugin makes outbound HTTP calls - for example datasette-auth-github or datasette-import-table - you may need to mock those HTTP requests in your tests.
The `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__ package is a useful library for mocking calls. It can be tricky to use with Datasette though since it mocks all HTTPX requests, and Datasette's own testing mechanism uses HTTPX internally.
To avoid breaking your tests, you can return ``["localhost"]`` from the ``non_mocked_hosts()`` fixture.
As an example, here's a very simple plugin which executes an HTTP response and returns the resulting content:
.. code-block:: python
from datasette import hookimpl
from datasette.utils.asgi import Response
import httpx
@hookimpl
def register_routes():
return [
(r"^/-/fetch-url$", fetch_url),
]
async def fetch_url(datasette, request):
if request.method == "GET":
return Response.html("""
<form action="/-/fetch-url" method="post">
<input name="url"><input type="submit">
</form>""")
vars = await request.post_vars()
url = vars["url"]
return Response.text(httpx.get(url).text)
Here's a test for that plugin that mocks the HTTPX outbound request:
.. code-block:: python
from datasette.app import Datasette
import pytest
@pytest.fixture
def non_mocked_hosts():
# This ensures httpx-mock will not affect Datasette's own
# httpx calls made in the tests by datasette.client:
return ["localhost"]
async def test_outbound_http_call(httpx_mock):
httpx_mock.add_response(
url="https://www.example.com/",
text="Hello world",
)
datasette = Datasette([], memory=True)
response = await datasette.client.post(
"/-/fetch-url",
data={"url": "https://www.example.com/"},
)
assert response.text == "Hello world"
outbound_request = httpx_mock.get_request()
assert (
outbound_request.url == "https://www.example.com/"
)
.. _testing_plugins_register_in_test: .. _testing_plugins_register_in_test:
Registering a plugin for the duration of a test Registering a plugin for the duration of a test

View file

@ -28,7 +28,7 @@ dependencies = [
"click-default-group>=1.2.3", "click-default-group>=1.2.3",
"Jinja2>=2.10.3", "Jinja2>=2.10.3",
"hupper>=1.9", "hupper>=1.9",
"httpx>=0.20,<1.0", "httpx2>=2.0",
"pluggy>=1.0", "pluggy>=1.0",
"uvicorn>=0.11", "uvicorn>=0.11",
"aiofiles>=0.4", "aiofiles>=0.4",

View file

@ -8,7 +8,7 @@ import tempfile
import time import time
from dataclasses import dataclass from dataclasses import dataclass
import httpx import httpx2
import pytest import pytest
import pytest_asyncio import pytest_asyncio
@ -32,13 +32,13 @@ UNDOCUMENTED_PERMISSIONS = {
} }
def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs): def wait_until_responds(url, timeout=5.0, client=httpx2, **kwargs):
start = time.time() start = time.time()
while time.time() - start < timeout: while time.time() - start < timeout:
try: try:
client.get(url, **kwargs) client.get(url, **kwargs)
return return
except httpx.ConnectError: except httpx2.ConnectError:
time.sleep(0.1) time.sleep(0.1)
raise AssertionError(f"Timed out waiting for {url} to respond") raise AssertionError(f"Timed out waiting for {url} to respond")
@ -277,8 +277,8 @@ def ds_unix_domain_socket_server(tmp_path_factory):
cwd=tempfile.gettempdir(), cwd=tempfile.gettempdir(),
) )
# Poll until available # Poll until available
transport = httpx.HTTPTransport(uds=uds) transport = httpx2.HTTPTransport(uds=uds)
client = httpx.Client(transport=transport) client = httpx2.Client(transport=transport)
try: try:
wait_until_responds( wait_until_responds(
"http://localhost/_memory.json", timeout=30.0, client=client "http://localhost/_memory.json", timeout=30.0, client=client

View file

@ -1,12 +1,12 @@
import socket import socket
import httpx import httpx2
import pytest import pytest
@pytest.mark.serial @pytest.mark.serial
def test_serve_localhost_http(ds_localhost_http_server): def test_serve_localhost_http(ds_localhost_http_server):
response = httpx.get("http://localhost:8041/_memory.json") response = httpx2.get("http://localhost:8041/_memory.json")
assert { assert {
"database": "_memory", "database": "_memory",
"path": "/_memory", "path": "/_memory",
@ -20,8 +20,8 @@ def test_serve_localhost_http(ds_localhost_http_server):
) )
def test_serve_unix_domain_socket(ds_unix_domain_socket_server): def test_serve_unix_domain_socket(ds_unix_domain_socket_server):
_, uds = ds_unix_domain_socket_server _, uds = ds_unix_domain_socket_server
transport = httpx.HTTPTransport(uds=uds) transport = httpx2.HTTPTransport(uds=uds)
client = httpx.Client(transport=transport) client = httpx2.Client(transport=transport)
response = client.get("http://localhost/_memory.json") response = client.get("http://localhost/_memory.json")
assert { assert {
"database": "_memory", "database": "_memory",

View file

@ -1,4 +1,4 @@
import httpx import httpx2
import pytest import pytest
import pytest_asyncio import pytest_asyncio
@ -43,7 +43,7 @@ async def datasette_with_permissions():
async def test_client_methods(datasette, method, path, expected_status): async def test_client_methods(datasette, method, path, expected_status):
client_method = getattr(datasette.client, method) client_method = getattr(datasette.client, method)
response = await client_method(path) response = await client_method(path)
assert isinstance(response, httpx.Response) assert isinstance(response, httpx2.Response)
assert response.status_code == expected_status assert response.status_code == expected_status
# Try that again using datasette.client.request # Try that again using datasette.client.request
response2 = await datasette.client.request(method, path) response2 = await datasette.client.request(method, path)
@ -63,7 +63,7 @@ async def test_client_post(datasette, prefix):
"message": "A message", "message": "A message",
}, },
) )
assert isinstance(response, httpx.Response) assert isinstance(response, httpx2.Response)
assert response.status_code == 302 assert response.status_code == 302
assert "ds_messages" in response.cookies assert "ds_messages" in response.cookies
finally: finally:
@ -135,7 +135,7 @@ async def test_skip_permission_checks_all_methods(datasette_with_permissions, me
response = await client_method("/test_db.json", skip_permission_checks=True) response = await client_method("/test_db.json", skip_permission_checks=True)
# We don't check status code since some methods might not be allowed, # We don't check status code since some methods might not be allowed,
# but we verify the request doesn't fail due to permissions # but we verify the request doesn't fail due to permissions
assert isinstance(response, httpx.Response) assert isinstance(response, httpx2.Response)
@pytest.mark.asyncio @pytest.mark.asyncio
@ -340,7 +340,7 @@ async def test_actor_parameter_all_http_methods(datasette, method):
client_method = getattr(datasette.client, method) client_method = getattr(datasette.client, method)
# Just verify no TypeError about unexpected 'actor' kwarg # Just verify no TypeError about unexpected 'actor' kwarg
response = await client_method("/", actor={"id": "root"}) response = await client_method("/", actor={"id": "root"})
assert isinstance(response, httpx.Response) assert isinstance(response, httpx2.Response)
@pytest.mark.asyncio @pytest.mark.asyncio

View file

@ -5,7 +5,7 @@ import subprocess
import sys import sys
import time import time
import httpx import httpx2
import pytest import pytest
from datasette.fixtures import write_fixture_database from datasette.fixtures import write_fixture_database
@ -34,11 +34,11 @@ def wait_for_server(process, url, timeout=30):
f"stderr:\n{stderr}" f"stderr:\n{stderr}"
) )
try: try:
response = httpx.get(url, timeout=1.0) response = httpx2.get(url, timeout=1.0)
if response.status_code < 500: if response.status_code < 500:
return return
last_error = f"HTTP {response.status_code}: {response.text[:200]}" last_error = f"HTTP {response.status_code}: {response.text[:200]}"
except httpx.HTTPError as ex: except httpx2.HTTPError as ex:
last_error = repr(ex) last_error = repr(ex)
time.sleep(0.1) time.sleep(0.1)
if process.poll() is None: if process.poll() is None:
@ -336,7 +336,7 @@ def project_rows(datasette_server, **filters):
"_shape": "objects", "_shape": "objects",
**{key: str(value) for key, value in filters.items()}, **{key: str(value) for key, value in filters.items()},
} }
response = httpx.get(f"{datasette_server}data/projects.json", params=params) response = httpx2.get(f"{datasette_server}data/projects.json", params=params)
response.raise_for_status() response.raise_for_status()
return response.json()["rows"] return response.json()["rows"]
@ -348,7 +348,7 @@ def project_row(datasette_server, pk):
def binary_file_blob(datasette_server, pk): def binary_file_blob(datasette_server, pk):
response = httpx.get( response = httpx2.get(
f"{datasette_server}data/binary_files/{pk}.blob", f"{datasette_server}data/binary_files/{pk}.blob",
params={"_blob_column": "data"}, params={"_blob_column": "data"},
) )
@ -369,7 +369,7 @@ def bulk_default_rows(datasette_server, **filters):
"_shape": "objects", "_shape": "objects",
**{key: str(value) for key, value in filters.items()}, **{key: str(value) for key, value in filters.items()},
} }
response = httpx.get(f"{datasette_server}data/bulk_defaults.json", params=params) response = httpx2.get(f"{datasette_server}data/bulk_defaults.json", params=params)
response.raise_for_status() response.raise_for_status()
return response.json()["rows"] return response.json()["rows"]
@ -379,7 +379,7 @@ def upsert_item_rows(datasette_server, **filters):
"_shape": "objects", "_shape": "objects",
**{key: str(value) for key, value in filters.items()}, **{key: str(value) for key, value in filters.items()},
} }
response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params) response = httpx2.get(f"{datasette_server}data/upsert_items.json", params=params)
response.raise_for_status() response.raise_for_status()
return response.json()["rows"] return response.json()["rows"]
@ -473,7 +473,7 @@ def test_create_table_flow(page, datasette_server):
page.wait_for_url("**/data/playwright_created") page.wait_for_url("**/data/playwright_created")
assert "playwright_created" in page.locator("h1").inner_text() assert "playwright_created" in page.locator("h1").inner_text()
response = httpx.get( response = httpx2.get(
f"{datasette_server}data/playwright_created.json?_extra=columns,column_types" f"{datasette_server}data/playwright_created.json?_extra=columns,column_types"
) )
response.raise_for_status() response.raise_for_status()
@ -487,7 +487,7 @@ def test_create_table_flow(page, datasette_server):
assert data["column_types"] == { assert data["column_types"] == {
"metadata": {"type": "json", "config": None}, "metadata": {"type": "json", "config": None},
} }
schema_response = httpx.get( schema_response = httpx2.get(
f"{datasette_server}data/-/query.json", f"{datasette_server}data/-/query.json",
params={ params={
"sql": ( "sql": (
@ -603,7 +603,7 @@ def test_create_table_from_data_flow(page, datasette_server):
dialog.locator(".table-create-save").click() dialog.locator(".table-create-save").click()
page.wait_for_url("**/data/playwright_from_data") page.wait_for_url("**/data/playwright_from_data")
response = httpx.get( response = httpx2.get(
f"{datasette_server}data/playwright_from_data.json?_shape=objects" f"{datasette_server}data/playwright_from_data.json?_shape=objects"
) )
response.raise_for_status() response.raise_for_status()
@ -639,7 +639,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
dialog.locator(".table-create-save").click() dialog.locator(".table-create-save").click()
page.wait_for_url("**/data/playwright_numeric_blanks") page.wait_for_url("**/data/playwright_numeric_blanks")
response = httpx.get( response = httpx2.get(
f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects" f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects"
) )
response.raise_for_status() response.raise_for_status()
@ -648,7 +648,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
{"name": "B", "score": None}, {"name": "B", "score": None},
] ]
schema_response = httpx.get( schema_response = httpx2.get(
f"{datasette_server}data/-/query.json", f"{datasette_server}data/-/query.json",
params={ params={
"sql": ( "sql": (
@ -856,7 +856,7 @@ def test_alter_table_flow(page, datasette_server):
columns = [] columns = []
for _ in range(20): for _ in range(20):
response = httpx.get(f"{datasette_server}data/projects.json?_extra=columns") response = httpx2.get(f"{datasette_server}data/projects.json?_extra=columns")
response.raise_for_status() response.raise_for_status()
columns = response.json()["columns"] columns = response.json()["columns"]
if "status" in columns: if "status" in columns: