Migrate from httpx to httpx2, closes #2879

https://claude.ai/code/session_01Xdqoneq8ddvruVZETo6rFf
This commit is contained in:
Simon Willison 2026-09-10 19:44:49 -07:00 committed by GitHub
commit b338c6f5f6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 86 additions and 86 deletions

View file

@ -28,7 +28,7 @@ import urllib.parse
from concurrent import futures
from pathlib import Path
import httpx
import httpx2
from itsdangerous import BadSignature, URLSafeSerializer
from jinja2 import (
ChoiceLoader,
@ -2995,7 +2995,7 @@ ORDER BY allowed.parent, allowed.child
This is the single entry point used by both AsgiLifespan (so
real deployments finish startup before accepting requests) and
AsgiRunOnFirstRequest (the fallback for hosts that never send
lifespan events, e.g. DatasetteClient's httpx.ASGITransport), and
lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and
`datasette serve` (cli.py) calls it too. The fast path below checks
both `_startup_invoked` and `_setup_db_done` - not just the former -
so that a bare `await ds.invoke_startup()` made by a caller ahead of
@ -3469,14 +3469,14 @@ class DatasetteClient:
with _DatasetteClientContext():
if skip_permission_checks:
with SkipPermissions():
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=self.app),
async with httpx2.AsyncClient(
transport=httpx2.ASGITransport(app=self.app),
cookies=kwargs.pop("cookies", None),
) as client:
return await getattr(client, method)(self._fix(path), **kwargs)
else:
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=self.app),
async with httpx2.AsyncClient(
transport=httpx2.ASGITransport(app=self.app),
cookies=kwargs.pop("cookies", None),
) as client:
return await getattr(client, method)(self._fix(path), **kwargs)
@ -3523,10 +3523,10 @@ class DatasetteClient:
method: HTTP method (e.g., "GET", "POST", "PUT")
path: The path to request
skip_permission_checks: If True, bypass all permission checks for this request
**kwargs: Additional arguments to pass to httpx
**kwargs: Additional arguments to pass to httpx2
Returns:
httpx.Response: The response from the request
httpx2.Response: The response from the request
"""
from datasette.permissions import SkipPermissions
@ -3535,16 +3535,16 @@ class DatasetteClient:
with _DatasetteClientContext():
if skip_permission_checks:
with SkipPermissions():
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=self.app),
async with httpx2.AsyncClient(
transport=httpx2.ASGITransport(app=self.app),
cookies=kwargs.pop("cookies", None),
) as client:
return await client.request(
method, self._fix(path, avoid_path_rewrites), **kwargs
)
else:
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=self.app),
async with httpx2.AsyncClient(
transport=httpx2.ASGITransport(app=self.app),
cookies=kwargs.pop("cookies", None),
) as client:
return await client.request(

View file

@ -4,7 +4,7 @@ from urllib.parse import urlencode
from asgiref.sync import async_to_sync
# These wrapper classes pre-date the introduction of
# datasette.client and httpx to Datasette. They could
# datasette.client and httpx2 to Datasette. They could
# be removed if the Datasette tests are modified to
# call datasette.client directly.

View file

@ -4,6 +4,13 @@
Changelog
=========
.. _unreleased:
Unreleased
----------
- Datasette now uses `httpx2 <https://httpx2.pydantic.dev/>`__, the Pydantic-maintained continuation of `httpx <https://www.python-httpx.org/>`__, in place of ``httpx``. The public API is the same, but responses returned by :ref:`internals_datasette_client` are now ``httpx2.Response`` objects rather than ``httpx.Response``. Plugins that use ``isinstance()`` checks against ``httpx.Response`` should be updated to use ``httpx2``. **Plugins that use httpx without explicitly depending on it** will need to add an explicit dependency or switch to `httpx2`.
.. _v1_0_a39:
1.0a39 (2026-09-10)

View file

@ -1594,32 +1594,32 @@ datasette.client
Plugins can make internal simulated HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins, while avoiding the overhead of making an external HTTP call to access those APIs.
The ``datasette.client`` object is a wrapper around the `HTTPX Python library <https://www.python-httpx.org/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
The ``datasette.client`` object is a wrapper around the `HTTPX2 Python library <https://httpx2.pydantic.dev/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
It offers the following methods:
``await datasette.client.get(path, **kwargs)`` - returns HTTPX Response
``await datasette.client.get(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal GET request against that path.
``await datasette.client.post(path, **kwargs)`` - returns HTTPX Response
``await datasette.client.post(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal POST request. Use ``data={"name": "value"}`` to pass form parameters.
``await datasette.client.options(path, **kwargs)`` - returns HTTPX Response
``await datasette.client.options(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal OPTIONS request.
``await datasette.client.head(path, **kwargs)`` - returns HTTPX Response
``await datasette.client.head(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal HEAD request.
``await datasette.client.put(path, **kwargs)`` - returns HTTPX Response
``await datasette.client.put(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal PUT request.
``await datasette.client.patch(path, **kwargs)`` - returns HTTPX Response
``await datasette.client.patch(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal PATCH request.
``await datasette.client.delete(path, **kwargs)`` - returns HTTPX Response
``await datasette.client.delete(path, **kwargs)`` - returns HTTPX2 Response
Execute an internal DELETE request.
``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX Response
``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX2 Response
Execute an internal request with the given HTTP method against that path.
These methods can be used with :ref:`internals_datasette_urls` - for example:
@ -1636,7 +1636,7 @@ These methods can be used with :ref:`internals_datasette_urls` - for example:
``datasette.client`` methods automatically take the current :ref:`setting_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path.
For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation <https://www.python-httpx.org/async/>`__.
For documentation on available ``**kwargs`` options and the shape of the HTTPX2 Response object refer to the `HTTPX2 Async documentation <https://httpx2.pydantic.dev/async/>`__.
.. _internals_datasette_client_actor:
@ -2630,12 +2630,12 @@ This example uses trace to record the start, end and duration of any HTTP GET re
.. code-block:: python
from datasette.tracer import trace
import httpx
import httpx2
async def fetch_url(url):
with trace("fetch-url", url=url):
async with httpx.AsyncClient() as client:
async with httpx2.AsyncClient() as client:
return await client.get(url)
.. _internals_tracer_trace_child_tasks:

View file

@ -25,7 +25,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl
)
This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX <https://www.python-httpx.org/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX2 <https://httpx2.pydantic.dev/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
This test also uses the `pytest-asyncio <https://pypi.org/project/pytest-asyncio/>`__ package to add support for ``async def`` test functions running under pytest.
@ -154,7 +154,7 @@ If you need to opt out of this behavior, add the following to your ``pytest.ini`
Using datasette.client in tests
-------------------------------
The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client <https://www.python-httpx.org/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX2 async client <https://httpx2.pydantic.dev/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
A simple test looks like this:
@ -273,22 +273,22 @@ If you want to create that test database repeatedly for every individual test fu
.. _testing_plugins_pytest_httpx:
Testing outbound HTTP calls with pytest-httpx
---------------------------------------------
Testing outbound HTTP calls with pytest-httpx2
----------------------------------------------
If your plugin makes outbound HTTP calls - for example datasette-auth-github or datasette-import-table - you may need to mock those HTTP requests in your tests.
The `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__ package is a useful library for mocking calls. It can be tricky to use with Datasette though since it mocks all HTTPX requests, and Datasette's own testing mechanism uses HTTPX internally.
The `pytest-httpx2 <https://pypi.org/project/pytest-httpx2/>`__ package provides a ``httpx2_mock`` fixture, built on `respx <https://lundberg.github.io/respx/>`__, for mocking outbound calls made using HTTPX2.
To avoid breaking your tests, you can return ``["localhost"]`` from the ``non_mocked_hosts()`` fixture.
Datasette's own ``datasette.client`` mechanism uses HTTPX2 internally too, but those requests are passed directly to the ASGI application rather than being sent over the network, so they are not affected by the mock.
As an example, here's a very simple plugin which executes an HTTP response and returns the resulting content:
As an example, here's a very simple plugin which executes an HTTP request and returns the resulting content:
.. code-block:: python
from datasette import hookimpl
from datasette.utils.asgi import Response
import httpx
import httpx2
@hookimpl
@ -306,27 +306,18 @@ As an example, here's a very simple plugin which executes an HTTP response and r
</form>""")
vars = await request.post_vars()
url = vars["url"]
return Response.text(httpx.get(url).text)
return Response.text(httpx2.get(url).text)
Here's a test for that plugin that mocks the HTTPX outbound request:
Here's a test for that plugin that mocks the HTTPX2 outbound request:
.. code-block:: python
from datasette.app import Datasette
import pytest
@pytest.fixture
def non_mocked_hosts():
# This ensures httpx-mock will not affect Datasette's own
# httpx calls made in the tests by datasette.client:
return ["localhost"]
async def test_outbound_http_call(httpx_mock):
httpx_mock.add_response(
url="https://www.example.com/",
text="Hello world",
async def test_outbound_http_call(httpx2_mock):
httpx2_mock.get("https://www.example.com/").respond(
text="Hello world"
)
datasette = Datasette([], memory=True)
response = await datasette.client.post(
@ -335,11 +326,13 @@ Here's a test for that plugin that mocks the HTTPX outbound request:
)
assert response.text == "Hello world"
outbound_request = httpx_mock.get_request()
outbound_request = httpx2_mock.calls.last.request
assert (
outbound_request.url == "https://www.example.com/"
)
If your plugin still makes its outbound calls using the original ``httpx`` library you can continue to mock those using `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__.
.. _testing_plugins_register_in_test:
Registering a plugin for the duration of a test

View file

@ -28,7 +28,7 @@ dependencies = [
"click-default-group>=1.2.3",
"Jinja2>=2.10.3",
"hupper>=1.9",
"httpx>=0.20,<1.0",
"httpx2>=2.0",
"pluggy>=1.0",
"uvicorn>=0.29",
"aiofiles>=0.4",

View file

@ -9,7 +9,7 @@ import tempfile
import time
from dataclasses import dataclass
import httpx
import httpx2
import pytest
import pytest_asyncio
@ -33,7 +33,7 @@ UNDOCUMENTED_PERMISSIONS = {
}
def wait_until_responds(url, timeout=5.0, client=httpx, process=None, **kwargs):
def wait_until_responds(url, timeout=5.0, client=httpx2, process=None, **kwargs):
start = time.time()
while time.time() - start < timeout:
# If the server died there is no point waiting out the timeout - fail
@ -47,7 +47,7 @@ def wait_until_responds(url, timeout=5.0, client=httpx, process=None, **kwargs):
try:
client.get(url, **kwargs)
return
except httpx.TransportError:
except httpx2.TransportError:
time.sleep(0.1)
raise AssertionError(f"Timed out waiting for {url} to respond")
@ -292,8 +292,8 @@ def ds_unix_domain_socket_server(tmp_path_factory):
cwd=tempfile.gettempdir(),
)
# Poll until available
transport = httpx.HTTPTransport(uds=uds)
client = httpx.Client(transport=transport)
transport = httpx2.HTTPTransport(uds=uds)
client = httpx2.Client(transport=transport)
try:
wait_until_responds(
"http://localhost/_memory.json", timeout=30.0, client=client

View file

@ -1,13 +1,13 @@
import socket
import time
import httpx
import httpx2
import pytest
@pytest.mark.serial
def test_serve_localhost_http(ds_localhost_http_server):
response = httpx.get("http://localhost:8041/_memory.json")
response = httpx2.get("http://localhost:8041/_memory.json")
assert {
"database": "_memory",
"path": "/_memory",
@ -21,8 +21,8 @@ def test_serve_localhost_http(ds_localhost_http_server):
)
def test_serve_unix_domain_socket(ds_unix_domain_socket_server):
_, uds = ds_unix_domain_socket_server
transport = httpx.HTTPTransport(uds=uds)
client = httpx.Client(transport=transport)
transport = httpx2.HTTPTransport(uds=uds)
client = httpx2.Client(transport=transport)
response = client.get("http://localhost/_memory.json")
assert {
"database": "_memory",
@ -97,7 +97,7 @@ def test_startup_hook_background_task_runs_on_serving_loop(serve_with_plugins):
deadline = time.time() + 3.0
payload = {}
while time.time() < deadline:
payload = httpx.get(
payload = httpx2.get(
f"http://127.0.0.1:{port}/-/marker-task-ran", timeout=1.0
).json()
if payload["marker_task_ran"]:

View file

@ -1,4 +1,4 @@
import httpx
import httpx2
import pytest
import pytest_asyncio
@ -43,7 +43,7 @@ async def datasette_with_permissions():
async def test_client_methods(datasette, method, path, expected_status):
client_method = getattr(datasette.client, method)
response = await client_method(path)
assert isinstance(response, httpx.Response)
assert isinstance(response, httpx2.Response)
assert response.status_code == expected_status
# Try that again using datasette.client.request
response2 = await datasette.client.request(method, path)
@ -63,7 +63,7 @@ async def test_client_post(datasette, prefix):
"message": "A message",
},
)
assert isinstance(response, httpx.Response)
assert isinstance(response, httpx2.Response)
assert response.status_code == 302
assert "ds_messages" in response.cookies
finally:
@ -135,7 +135,7 @@ async def test_skip_permission_checks_all_methods(datasette_with_permissions, me
response = await client_method("/test_db.json", skip_permission_checks=True)
# We don't check status code since some methods might not be allowed,
# but we verify the request doesn't fail due to permissions
assert isinstance(response, httpx.Response)
assert isinstance(response, httpx2.Response)
@pytest.mark.asyncio
@ -340,7 +340,7 @@ async def test_actor_parameter_all_http_methods(datasette, method):
client_method = getattr(datasette.client, method)
# Just verify no TypeError about unexpected 'actor' kwarg
response = await client_method("/", actor={"id": "root"})
assert isinstance(response, httpx.Response)
assert isinstance(response, httpx2.Response)
@pytest.mark.asyncio

View file

@ -5,7 +5,7 @@ into the ASGI lifespan protocol.
These exercise Datasette._startup_sequence() via three different callers:
- AsgiLifespan, by hand-driving lifespan.startup messages (no HTTP request)
- AsgiRunOnFirstRequest, the fallback for hosts that never send lifespan
events (this is what DatasetteClient / plain httpx.ASGITransport uses)
events (this is what DatasetteClient / plain httpx2.ASGITransport uses)
- Both at once, to prove startup hooks run at most once
"""
@ -13,7 +13,7 @@ import asyncio
import contextlib
import sqlite3
import httpx
import httpx2
import pytest
from datasette import hookimpl
@ -131,8 +131,8 @@ async def test_startup_runs_exactly_once_across_lifespan_and_first_request():
# A first HTTP request (as if the host never sent lifespan events,
# or lifespan already ran) should not run the hook again.
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(
transport = httpx2.ASGITransport(app=app)
async with httpx2.AsyncClient(
transport=transport, base_url="http://localhost"
) as client:
response1 = await client.get("/-/versions.json")
@ -149,13 +149,13 @@ async def test_startup_runs_exactly_once_across_lifespan_and_first_request():
@pytest.mark.asyncio
async def test_no_lifespan_first_request_still_triggers_startup():
# Pin today's behavior: a client that never drives ASGI lifespan events
# at all (like httpx.ASGITransport, which DatasetteClient uses) still
# at all (like httpx2.ASGITransport, which DatasetteClient uses) still
# gets startup armed by the AsgiRunOnFirstRequest fallback.
ds = Datasette(memory=True)
assert ds._startup_invoked is False
app = ds.app()
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(
transport = httpx2.ASGITransport(app=app)
async with httpx2.AsyncClient(
transport=transport, base_url="http://localhost"
) as client:
response = await client.get("/-/versions.json")
@ -197,8 +197,8 @@ async def test_concurrent_first_requests_all_wait_for_slow_startup():
pm.register(SlowStartupPlugin(), name="slow_startup_plugin")
try:
app = ds.app()
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(
transport = httpx2.ASGITransport(app=app)
async with httpx2.AsyncClient(
transport=transport, base_url="http://localhost"
) as client:
responses = await asyncio.gather(

View file

@ -5,7 +5,7 @@ import subprocess
import sys
import time
import httpx
import httpx2
import pytest
from datasette.fixtures import write_fixture_database
@ -34,11 +34,11 @@ def wait_for_server(process, url, timeout=30):
f"stderr:\n{stderr}"
)
try:
response = httpx.get(url, timeout=1.0)
response = httpx2.get(url, timeout=1.0)
if response.status_code < 500:
return
last_error = f"HTTP {response.status_code}: {response.text[:200]}"
except httpx.HTTPError as ex:
except httpx2.HTTPError as ex:
last_error = repr(ex)
time.sleep(0.1)
if process.poll() is None:
@ -336,7 +336,7 @@ def project_rows(datasette_server, **filters):
"_shape": "objects",
**{key: str(value) for key, value in filters.items()},
}
response = httpx.get(f"{datasette_server}data/projects.json", params=params)
response = httpx2.get(f"{datasette_server}data/projects.json", params=params)
response.raise_for_status()
return response.json()["rows"]
@ -348,7 +348,7 @@ def project_row(datasette_server, pk):
def binary_file_blob(datasette_server, pk):
response = httpx.get(
response = httpx2.get(
f"{datasette_server}data/binary_files/{pk}.blob",
params={"_blob_column": "data"},
)
@ -369,7 +369,7 @@ def bulk_default_rows(datasette_server, **filters):
"_shape": "objects",
**{key: str(value) for key, value in filters.items()},
}
response = httpx.get(f"{datasette_server}data/bulk_defaults.json", params=params)
response = httpx2.get(f"{datasette_server}data/bulk_defaults.json", params=params)
response.raise_for_status()
return response.json()["rows"]
@ -379,7 +379,7 @@ def upsert_item_rows(datasette_server, **filters):
"_shape": "objects",
**{key: str(value) for key, value in filters.items()},
}
response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params)
response = httpx2.get(f"{datasette_server}data/upsert_items.json", params=params)
response.raise_for_status()
return response.json()["rows"]
@ -473,7 +473,7 @@ def test_create_table_flow(page, datasette_server):
page.wait_for_url("**/data/playwright_created")
assert "playwright_created" in page.locator("h1").inner_text()
response = httpx.get(
response = httpx2.get(
f"{datasette_server}data/playwright_created.json?_extra=columns,column_types"
)
response.raise_for_status()
@ -487,7 +487,7 @@ def test_create_table_flow(page, datasette_server):
assert data["column_types"] == {
"metadata": {"type": "json", "config": None},
}
schema_response = httpx.get(
schema_response = httpx2.get(
f"{datasette_server}data/-/query.json",
params={
"sql": (
@ -603,7 +603,7 @@ def test_create_table_from_data_flow(page, datasette_server):
dialog.locator(".table-create-save").click()
page.wait_for_url("**/data/playwright_from_data")
response = httpx.get(
response = httpx2.get(
f"{datasette_server}data/playwright_from_data.json?_shape=objects"
)
response.raise_for_status()
@ -639,7 +639,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
dialog.locator(".table-create-save").click()
page.wait_for_url("**/data/playwright_numeric_blanks")
response = httpx.get(
response = httpx2.get(
f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects"
)
response.raise_for_status()
@ -648,7 +648,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
{"name": "B", "score": None},
]
schema_response = httpx.get(
schema_response = httpx2.get(
f"{datasette_server}data/-/query.json",
params={
"sql": (
@ -856,7 +856,7 @@ def test_alter_table_flow(page, datasette_server):
columns = []
for _ in range(20):
response = httpx.get(f"{datasette_server}data/projects.json?_extra=columns")
response = httpx2.get(f"{datasette_server}data/projects.json?_extra=columns")
response.raise_for_status()
columns = response.json()["columns"]
if "status" in columns: