url links follow the page's truncate_cells, row pages keep the full url

tests/conftest.py

@@ -1,15 +1,18 @@
-import httpx
 import importlib.metadata
 import os
 import pathlib
-import pytest
-import pytest_asyncio
 import re
+import socket
 import subprocess
 import sys
 import tempfile
 import time
 from dataclasses import dataclass
+
+import httpx2
+import pytest
+import pytest_asyncio
+
 from datasette import Event, hookimpl
 
 try:
@@ -30,15 +33,39 @@ UNDOCUMENTED_PERMISSIONS = {
 }
 
 
-def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs):
+def wait_until_responds(url, timeout=5.0, client=httpx2, process=None, **kwargs):
     start = time.time()
     while time.time() - start < timeout:
+        # If the server died there is no point waiting out the timeout - fail
+        # now, with its output, instead of after `timeout` seconds of silence
+        if process is not None and process.poll() is not None:
+            raise AssertionError(
+                "Server exited early with returncode {}\n{}".format(
+                    process.returncode, process.stdout.read().decode("utf-8")
+                )
+            )
         try:
             client.get(url, **kwargs)
             return
-        except httpx.ConnectError:
+        except httpx2.TransportError:
             time.sleep(0.1)
-    raise AssertionError("Timed out waiting for {} to respond".format(url))
+    raise AssertionError(f"Timed out waiting for {url} to respond")
+
+
+def find_free_port():
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
+from datasette.telemetry_testing import (  # noqa: F401
+    MetricsCollector,
+    otel_meter_provider,
+    otel_metrics,
+    otel_provider,
+    otel_reset,
+    otel_spans,
+)
 
 
 @pytest.fixture
@@ -54,10 +81,12 @@ def bare_ds():
 
 @pytest_asyncio.fixture(scope="session")
 async def ds_client():
+    import secrets
+
     from datasette.app import Datasette
     from datasette.database import Database
+
     from .fixtures import CONFIG, METADATA, PLUGINS_DIR
-    import secrets
 
     ds = Datasette(
         metadata=METADATA,
@@ -87,7 +116,10 @@ async def ds_client():
 
     await db.execute_write_fn(prepare)
     await ds.invoke_startup()
-    return ds.client
+    try:
+        yield ds.client
+    finally:
+        ds.close()
 
 
 def pytest_report_header(config):
@@ -96,8 +128,8 @@ def pytest_report_header(config):
     conn.close()
     sqlite_utils_version = importlib.metadata.version("sqlite-utils")
     headers = [
-        "SQLite: {}".format(version),
-        "sqlite-utils: {}".format(sqlite_utils_version),
+        f"SQLite: {version}",
+        f"sqlite-utils: {sqlite_utils_version}",
     ]
     if config.getoption("--playwright"):
         try:
@@ -149,6 +181,11 @@ def pytest_collection_modifyitems(config, items):
     move_to_front(items, "test_spatialite_error_if_attempt_to_open_spatialite")
     move_to_front(items, "test_package")
     move_to_front(items, "test_package_with_port")
+    # These start subprocesses, which can crash on macOS/CPython 3.13 late in
+    # a test run once the pytest process has started many threads
+    move_to_front(items, "test_datasette_package_never_imports_the_sdk")
+    move_to_front(items, "test_kit_module_itself_never_imports_the_sdk")
+    move_to_front(items, "test_no_provider_takes_the_fast_path")
 
 
 def move_to_front(items, test_name):
@@ -175,8 +212,8 @@ def restore_working_directory(tmpdir, request):
 
 @pytest.fixture(scope="session", autouse=True)
 def check_actions_are_documented():
-    from datasette.plugins import pm
     from datasette.default_actions import register_actions as default_register_actions
+    from datasette.plugins import pm
 
     content = (
         pathlib.Path(__file__).parent.parent / "docs" / "authentication.rst"
@@ -202,7 +239,7 @@ def check_actions_are_documented():
             if kwargs["action"] in core_actions:
                 assert (
                     action in documented_actions
-                ), "Undocumented permission action: {}".format(action)
+                ), f"Undocumented permission action: {action}"
 
     pm.add_hookcall_monitoring(
         before=before, after=lambda outcome, hook_name, hook_impls, kwargs: None
@@ -247,12 +284,24 @@ def ds_localhost_http_server():
         # Avoid FileNotFoundError: [Errno 2] No such file or directory:
         cwd=tempfile.gettempdir(),
     )
-    wait_until_responds("http://localhost:8041/")
-    # Check it started successfully
-    assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8")
-    yield ds_proc
-    # Shut it down at the end of the pytest session
-    ds_proc.terminate()
+    try:
+        wait_until_responds("http://localhost:8041/", process=ds_proc)
+        yield ds_proc
+    finally:
+        stop_process(ds_proc)
+
+
+def stop_process(proc):
+    try:
+        if proc.poll() is None:
+            proc.terminate()
+        try:
+            proc.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait()
+    finally:
+        proc.stdout.close()
 
 
 @pytest.fixture(scope="session")
@@ -273,11 +322,27 @@ def ds_unix_domain_socket_server(tmp_path_factory):
         cwd=tempfile.gettempdir(),
     )
     # Poll until available
-    transport = httpx.HTTPTransport(uds=uds)
-    client = httpx.Client(transport=transport)
+    transport = httpx2.HTTPTransport(uds=uds)
+    client = httpx2.Client(transport=transport)
     try:
+        # Probe with a socket we own: the HTTP transport can leak a socket
+        # when connect() fails before the UDS server has started listening.
+        start = time.monotonic()
+        while True:
+            try:
+                with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as probe:
+                    probe.settimeout(0.1)
+                    probe.connect(uds)
+                break
+            except OSError:
+                if ds_proc.poll() is not None or time.monotonic() - start > 30:
+                    raise
+                time.sleep(0.1)
         wait_until_responds(
-            "http://localhost/_memory.json", timeout=30.0, client=client
+            "http://localhost/_memory.json",
+            timeout=30.0,
+            client=client,
+            process=ds_proc,
         )
         # Check it started successfully
         assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8")
@@ -285,20 +350,75 @@ def ds_unix_domain_socket_server(tmp_path_factory):
     finally:
         client.close()
         # Shut it down at the end of the pytest session
-        ds_proc.terminate()
-        try:
-            ds_proc.wait(timeout=5)
-        except subprocess.TimeoutExpired:
-            ds_proc.kill()
-            ds_proc.wait()
+        stop_process(ds_proc)
         try:
             os.unlink(uds)
         except FileNotFoundError:
             pass
 
 
+@pytest.fixture
+def serve_with_plugins(tmp_path):
+    """Factory fixture for starting ``datasette serve`` in a subprocess with
+    plugins written to a temporary ``--plugins-dir``.
+
+    For tests that need the real serve path: event-loop wiring, exit codes,
+    signals. The usual in-process ``pm.register`` plugin pattern can't reach
+    a subprocess, so plugin source is written out as importable files instead.
+
+    Unlike ``ds_localhost_http_server`` this is function-scoped and takes a
+    fresh port each time, because each test needs its own plugins. Call it as::
+
+        proc, port = serve_with_plugins({"my_plugin": PLUGIN_SOURCE})
+
+    ``plugins`` maps module name to Python source. Pass
+    ``wait_for_startup=False`` when the server is expected to fail during
+    startup rather than begin serving. Extra CLI arguments are passed through.
+    Every process started is terminated when the test ends.
+    """
+    processes = []
+
+    def start(plugins, *extra_args, wait_for_startup=True):
+        plugins_dir = tmp_path / "plugins"
+        plugins_dir.mkdir(exist_ok=True)
+        for module_name, source in plugins.items():
+            (plugins_dir / f"{module_name}.py").write_text(source, "utf-8")
+        port = find_free_port()
+        proc = subprocess.Popen(
+            [
+                sys.executable,
+                "-m",
+                "datasette",
+                "--memory",
+                "--plugins-dir",
+                str(plugins_dir),
+                "-h",
+                "127.0.0.1",
+                "-p",
+                str(port),
+                *extra_args,
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            # Avoid FileNotFoundError: [Errno 2] No such file or directory:
+            cwd=tempfile.gettempdir(),
+        )
+        processes.append(proc)
+        if wait_for_startup:
+            wait_until_responds(
+                f"http://127.0.0.1:{port}/-/versions.json", process=proc
+            )
+        return proc, port
+
+    yield start
+
+    for proc in processes:
+        stop_process(proc)
+
+
 # Import fixtures from fixtures.py to make them available
-from .fixtures import (  # noqa: E402, F401
+from .fixtures import (  # noqa: F401
+    TEMP_PLUGIN_SECRET_FILE,
     app_client,
     app_client_base_url_prefix,
     app_client_conflicting_database_names,
@@ -315,5 +435,4 @@ from .fixtures import (  # noqa: E402, F401
     app_client_with_dot,
     app_client_with_trace,
     make_app_client,
-    TEMP_PLUGIN_SECRET_FILE,
 )

tests/fixtures.py

@@ -1,3 +1,13 @@
+import contextlib
+import json
+import os
+import pathlib
+import tempfile
+import textwrap
+
+import click
+import pytest
+
 from datasette.app import Datasette
 from datasette.fixtures import (
     EXTRA_DATABASE_SQL,
@@ -5,14 +15,6 @@ from datasette.fixtures import (
     write_fixture_database,
 )
 from datasette.utils.testing import TestClient
-import click
-import contextlib
-import json
-import os
-import pathlib
-import pytest
-import tempfile
-import textwrap
 
 # This temp file is used by one of the plugin config tests
 TEMP_PLUGIN_SECRET_FILE = os.path.join(tempfile.gettempdir(), "plugin-secret")
@@ -167,12 +169,10 @@ def make_app_client(
             template_dir=template_dir,
             crossdb=crossdb,
         )
-        yield TestClient(ds)
-        # Close as many database connections as possible
-        # to try and avoid too many open files error
-        for db in ds.databases.values():
-            if not db.is_memory:
-                db.close()
+        try:
+            yield TestClient(ds)
+        finally:
+            ds.close()
 
 
 @pytest.fixture(scope="session")
@@ -184,9 +184,10 @@ def app_client():
 @pytest.fixture(scope="session")
 def app_client_no_files():
     ds = Datasette([])
-    yield TestClient(ds)
-    for db in ds.databases.values():
-        db.close()
+    try:
+        yield TestClient(ds)
+    finally:
+        ds.close()
 
 
 @pytest.fixture(scope="session")

tests/plugins/my_plugin.py

@@ -1,16 +1,16 @@
 import asyncio
-from datasette import hookimpl
-from datasette.facets import Facet
-from datasette.tokens import TokenHandler
-from datasette import tracer
-from datasette.permissions import Action
-from datasette.resources import DatabaseResource
-from datasette.utils import path_with_added_args
-from datasette.utils.asgi import asgi_send_json, Response
 import base64
 import json
 import urllib.parse
 
+from datasette import hookimpl, tracer
+from datasette.facets import Facet
+from datasette.permissions import Action
+from datasette.resources import DatabaseResource
+from datasette.tokens import TokenHandler
+from datasette.utils import path_with_added_args
+from datasette.utils.asgi import Response, asgi_send_json
+
 
 @hookimpl
 def prepare_connection(conn, database, datasette):
@@ -305,11 +305,7 @@ def startup(datasette):
     datasette._startup_hook_fired = True
 
     # And test some import shortcuts too
-    from datasette import Response
-    from datasette import Forbidden
-    from datasette import NotFound
-    from datasette import hookimpl
-    from datasette import actor_matches_allow
+    from datasette import Forbidden, NotFound, Response, actor_matches_allow, hookimpl
 
     _ = (Response, Forbidden, NotFound, hookimpl, actor_matches_allow)
 
@@ -373,7 +369,7 @@ def table_actions(datasette, database, table, actor, request):
                     "label": "Plugin button",
                     "description": "Runs JavaScript from a plugin",
                     "attrs": {
-                        "aria-label": "Plugin button for {}".format(table),
+                        "aria-label": f"Plugin button for {table}",
                         "data-plugin-action": "plugin-button",
                         "data-database": database,
                         "data-table": table,

tests/plugins/my_plugin_2.py

@@ -1,8 +1,10 @@
+import json
+from functools import wraps
+
+import markupsafe
+
 from datasette import hookimpl
 from datasette.utils.asgi import Response
-from functools import wraps
-import markupsafe
-import json
 
 
 @hookimpl
@@ -33,11 +35,7 @@ def render_cell(value, database):
     if set(data.keys()) != {"href", "label"}:
         return None
     href = data["href"]
-    if not (
-        href.startswith("/")
-        or href.startswith("http://")
-        or href.startswith("https://")
-    ):
+    if not (href.startswith(("/", "http://", "https://"))):
         return None
     return markupsafe.Markup(
         '<a data-database="{database}" href="{href}">{label}</a>'.format(
@@ -54,7 +52,7 @@ def extra_template_vars(template, database, table, view_name, request, datasette
     datasette._last_request = request
 
     async def query_database(sql):
-        first_db = list(datasette.databases.keys())[0]
+        first_db = next(iter(datasette.databases.keys()))
         return (await datasette.execute(first_db, sql)).rows[0][0]
 
     async def inner():
@@ -172,10 +170,10 @@ def register_routes(datasette):
     path = config["path"]
 
     def new_table(request):
-        return Response.text("/db/table: {}".format(sorted(request.url_vars.items())))
+        return Response.text(f"/db/table: {sorted(request.url_vars.items())}")
 
     return [
-        (r"/{}/$".format(path), lambda: Response.text(path.upper())),
+        (rf"/{path}/$", lambda: Response.text(path.upper())),
         # Also serves to demonstrate over-ride of default paths:
         (r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)", new_table),
     ]

tests/plugins/register_output_renderer.py

@@ -1,6 +1,7 @@
+import json
+
 from datasette import hookimpl
 from datasette.utils.asgi import Response
-import json
 
 
 async def can_render(
@@ -18,9 +19,7 @@ async def can_render(
         "request": request,
         "view_name": view_name,
     }
-    if request.args.get("_no_can_render"):
-        return False
-    return True
+    return not request.args.get("_no_can_render")
 
 
 async def render_test_all_parameters(

tests/plugins/sleep_sql_function.py

@@ -1,6 +1,7 @@
-from datasette import hookimpl
 import time
 
+from datasette import hookimpl
+
 
 @hookimpl
 def prepare_connection(conn):

tests/test_actions_sql.py

@@ -10,10 +10,11 @@ These tests verify:
 
 import pytest
 import pytest_asyncio
+
+from datasette import hookimpl
 from datasette.app import Datasette
 from datasette.permissions import PermissionSQL
 from datasette.resources import DatabaseResource, QueryResource, TableResource
-from datasette import hookimpl
 
 
 def test_resource_string_representations():
@@ -90,7 +91,7 @@ async def test_allowed_resources_global_allow(test_ds):
         assert all(isinstance(t, TableResource) for t in tables)
 
         # Check specific tables are present
-        table_set = set((t.parent, t.child) for t in tables)
+        table_set = {(t.parent, t.child) for t in tables}
         assert ("analytics", "events") in table_set
         assert ("analytics", "users") in table_set
         assert ("analytics", "sensitive") in table_set

tests/test_actor_restriction_bug.py

@@ -6,6 +6,7 @@ config allow blocks can bypass table-level restrictions.
 """
 
 import pytest
+
 from datasette.app import Datasette
 from datasette.resources import TableResource
 

tests/test_allowed_many.py

@@ -10,6 +10,8 @@ Layer 3: table/database views precompute all registered actions before
 
 import pytest
 import pytest_asyncio
+
+from datasette import hookimpl
 from datasette.app import Datasette
 from datasette.permissions import (
     Action,
@@ -18,7 +20,6 @@ from datasette.permissions import (
     _permission_check_cache,
 )
 from datasette.resources import DatabaseResource, TableResource
-from datasette import hookimpl
 
 
 class CountingRulesPlugin:
@@ -114,7 +115,7 @@ async def test_allowed_not_memoized_without_cache(counting_ds):
 async def test_cache_keyed_on_full_actor_identity(counting_ds):
     """Interleaved checks for different actors never share cache entries."""
     # Uses drop-table because default permissions deny it to non-root actors
-    ds, plugin = counting_ds
+    ds, _plugin = counting_ds
     resource = TableResource("analytics", "users")
     token = _permission_check_cache.set({})
     try:
@@ -180,7 +181,7 @@ async def test_cache_keyed_on_resource(counting_ds):
 
 @pytest.mark.asyncio
 async def test_skip_permission_checks_bypasses_cache(counting_ds):
-    ds, plugin = counting_ds
+    ds, _plugin = counting_ds
     resource = TableResource("analytics", "users")
     token = _permission_check_cache.set({})
     try:

tests/test_allowed_resources.py

@@ -7,9 +7,10 @@ based on permission rules from plugins and configuration.
 
 import pytest
 import pytest_asyncio
+
+from datasette import hookimpl
 from datasette.app import Datasette
 from datasette.permissions import PermissionSQL
-from datasette import hookimpl
 
 
 # Test plugin that provides permission rules

tests/test_api.py

@@ -1,13 +1,16 @@
+import pathlib
+import urllib
+
+import pytest
+
 from datasette.app import Datasette
 from datasette.plugins import DEFAULT_PLUGINS
-from datasette.utils import UNSTABLE_API_MESSAGE
+from datasette.resources import DatabaseResource, TableResource
+from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode
 from datasette.utils.sqlite import sqlite_version
 from datasette.version import __version__
-from .fixtures import make_app_client, EXPECTED_PLUGINS
-import pathlib
-import pytest
-import sys
-import urllib
+
+from .fixtures import EXPECTED_PLUGINS, make_app_client
 
 
 @pytest.mark.asyncio
@@ -16,7 +19,7 @@ async def test_homepage(ds_client):
     assert response.status_code == 200
     assert "application/json; charset=utf-8" == response.headers["content-type"]
     data = response.json()
-    assert sorted(list(data.get("metadata").keys())) == [
+    assert sorted(data.get("metadata").keys()) == [
         "about",
         "about_url",
         "description_html",
@@ -99,14 +102,11 @@ async def test_database_page(ds_client):
         "tags",
     }
 
-    # Expected hidden tables
+    # The external-content index is visible, but its shadow tables need a
+    # second dependency hop and are excluded by the one-hop permission policy.
     expected_hidden_tables = {
         "no_primary_key",
         "searchable_fts",
-        "searchable_fts_config",
-        "searchable_fts_data",
-        "searchable_fts_docsize",
-        "searchable_fts_idx",
     }
 
     # Verify all expected tables exist
@@ -384,9 +384,7 @@ async def test_row_pk_arity_mismatch_returns_400(ds_client, row_path, suffix):
     # because the SQL had one bind placeholder per PK column but params were
     # only bound for the supplied components. It should be a 400 instead,
     # mirroring the existing guard in datasette/views/table.py.
-    response = await ds_client.get(
-        "/fixtures/compound_primary_key/{}{}".format(row_path, suffix)
-    )
+    response = await ds_client.get(f"/fixtures/compound_primary_key/{row_path}{suffix}")
     assert response.status_code == 400
     if suffix == ".json":
         assert response.json()["ok"] is False
@@ -458,6 +456,67 @@ async def test_row_foreign_key_tables(ds_client):
     ]
 
 
+@pytest.mark.asyncio
+async def test_row_foreign_key_tables_omit_denied_tables(request):
+    actor = {"id": "reader"}
+    ds = Datasette(
+        memory=True,
+        default_deny=True,
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "parents": {"permissions": {"view-table": True}},
+                        "private_children": {"permissions": {"view-table": False}},
+                    }
+                }
+            }
+        },
+    )
+    request.addfinalizer(ds.close)
+    db = ds.add_memory_database("fk_count_leak", name="data")
+    await db.execute_write("create table parents (id integer primary key, name text)")
+    await db.execute_write("""
+        create table private_children (
+            id integer primary key,
+            parent_id integer references parents(id)
+        )
+    """)
+    await db.execute_write("insert into parents values (1, 'Public parent')")
+    await db.execute_write("""
+        insert into private_children (id, parent_id) values
+            (1, 1),
+            (2, 1),
+            (3, 1)
+    """)
+    await ds.invoke_startup()
+
+    parent = TableResource(database="data", table="parents")
+    private_children = TableResource(database="data", table="private_children")
+    assert await ds.allowed(action="view-table", resource=parent, actor=actor)
+    assert not await ds.allowed(
+        action="view-table", resource=private_children, actor=actor
+    )
+    assert not await ds.allowed(
+        action="execute-sql",
+        resource=DatabaseResource(database="data"),
+        actor=actor,
+    )
+
+    direct_child = await ds.client.get("/data/private_children.json", actor=actor)
+    assert direct_child.status_code == 403
+    parent_response = await ds.client.get(
+        "/data/parents/1.json?_extra=foreign_key_tables", actor=actor
+    )
+    assert parent_response.status_code == 200
+
+    foreign_key_tables = parent_response.json().get("foreign_key_tables", [])
+    assert foreign_key_tables == [], (
+        "denied child table name, foreign-key column, and row count disclosed: "
+        f"{foreign_key_tables}"
+    )
+
+
 @pytest.mark.asyncio
 async def test_row_extras(ds_client):
     response = await ds_client.get(
@@ -600,8 +659,7 @@ async def test_threads_json(ds_client):
     finally:
         ds_client.ds.root_enabled = False
     expected_keys = {"ok", "threads", "num_threads"}
-    if sys.version_info >= (3, 7, 0):
-        expected_keys.update({"tasks", "num_tasks"})
+    expected_keys.update({"tasks", "num_tasks"})
     data = response.json()
     assert set(data.keys()) == expected_keys
     # Should be at least one _execute_writes thread for __INTERNAL__
@@ -614,13 +672,13 @@ async def test_plugins_json(ds_client):
     response = await ds_client.get("/-/plugins.json")
     # Filter out TrackEventPlugin
     actual_plugins = sorted(
-        [p for p in response.json()["plugins"] if p["name"] != "TrackEventPlugin"],
+        [p for p in response.json() if p["name"] != "TrackEventPlugin"],
         key=lambda p: p["name"],
     )
     assert EXPECTED_PLUGINS == actual_plugins
     # Try with ?all=1
     response = await ds_client.get("/-/plugins.json?all=1")
-    names = {p["name"] for p in response.json()["plugins"]}
+    names = {p["name"] for p in response.json()}
     assert names.issuperset(p["name"] for p in EXPECTED_PLUGINS)
     assert names.issuperset(DEFAULT_PLUGINS)
 
@@ -895,10 +953,7 @@ async def test_hidden_sqlite_stat1_table():
     await db.execute_write("analyze")
     data = (await ds.client.get("/db.json?_show_hidden=1")).json()
     tables = [(t["name"], t["hidden"]) for t in data["tables"]]
-    assert tables in (
-        [("normal", False), ("sqlite_stat1", True)],
-        [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)],
-    )
+    assert tables == [("normal", False)]
 
 
 @pytest.mark.asyncio
@@ -930,6 +985,33 @@ async def test_tilde_encoded_database_names(db_name):
     assert response2.status_code == 200
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("table_name", ("[foo]", "foo]", "[foo]/bar"))
+async def test_table_with_reserved_characters_in_name(table_name):
+    # Table names containing characters such as "]" that cannot be escaped
+    # using SQLite [bracket] quoting used to break schema introspection and
+    # the table page - https://github.com/simonw/datasette/issues/2431
+    ds = Datasette()
+    db = ds.add_memory_database("test_reserved_table_names")
+    await db.execute_write(
+        f"create table {escape_sqlite(table_name)} (id integer primary key, name text)"
+    )
+    await db.execute_write(
+        f"insert into {escape_sqlite(table_name)} (id, name) values (1, 'one')"
+    )
+    # Schema introspection (populate_schema_tables) must not crash:
+    db_response = await ds.client.get("/test_reserved_table_names.json")
+    assert db_response.status_code == 200
+    tables = {t["name"]: t for t in db_response.json()["tables"]}
+    assert tables[table_name]["count"] == 1
+    # And the table page itself must load and return the row:
+    table_response = await ds.client.get(
+        f"/test_reserved_table_names/{tilde_encode(table_name)}.json?_shape=array"
+    )
+    assert table_response.status_code == 200
+    assert table_response.json() == [{"id": 1, "name": "one"}]
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "config,expected",

tests/test_api_write.py

@@ -1,21 +1,25 @@
+import time
+
+import pytest
+import sqlite_utils
+
 from datasette.app import Datasette
 from datasette.events import RenameTableEvent
 from datasette.utils import error_body, escape_sqlite, sqlite3
+
 from .utils import last_event
-import pytest
-import time
 
 
 def assert_schema_contains(fragment, schema):
-    assert fragment in schema, "Expected schema to contain {!r}, got {!r}".format(
-        fragment, schema
-    )
+    assert (
+        fragment in schema
+    ), f"Expected schema to contain {fragment!r}, got {schema!r}"
 
 
 def assert_schema_not_contains(fragment, schema):
     assert (
         fragment not in schema
-    ), "Expected schema not to contain {!r}, got {!r}".format(fragment, schema)
+    ), f"Expected schema not to contain {fragment!r}, got {schema!r}"
 
 
 @pytest.fixture
@@ -47,17 +51,114 @@ def write_token(ds, actor_id="root", permissions=None):
 
 def _headers(token):
     return {
-        "Authorization": "Bearer {}".format(token),
+        "Authorization": f"Bearer {token}",
         "Content-Type": "application/json",
     }
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("operation", ["read", "read_row", "rename"])
+async def test_trailing_lf_table_permissions(tmp_path, operation):
+    # SQLite treats "secret" and "secret\n" as different table names. Permission
+    # checks and SQL execution must agree on which table a request targets.
+    db_path = tmp_path / "data.db"
+    conn = sqlite3.connect(str(db_path))
+    conn.executescript(
+        "create table secret (id integer primary key, value text);"
+        "insert into secret values (1, 'private');"
+    )
+    conn.close()
+    # Allow builder to create and use tables generally, but explicitly deny
+    # access to the existing secret table below. Disable arbitrary SQL access.
+    grants = {
+        action: {"id": "builder"}
+        for action in (
+            "view-database",
+            "create-table",
+            "view-table",
+            "insert-row",
+            "alter-table",
+        )
+    }
+    ds = Datasette(
+        [str(db_path)],
+        default_deny=True,
+        settings={"default_allow_sql": False},
+        config={
+            "permissions": {"view-instance": {"id": "builder"}},
+            "databases": {
+                "data": {
+                    "permissions": grants,
+                    "tables": {
+                        "secret": {
+                            "permissions": {
+                                "view-table": False,
+                                "insert-row": False,
+                                "alter-table": False,
+                            }
+                        }
+                    },
+                }
+            },
+        },
+    )
+    headers = _headers(write_token(ds, actor_id="builder"))
+    try:
+        # Establish that the protected table is inaccessible before creating
+        # a second table whose name differs only by a trailing line feed.
+        response = await ds.client.get("/data/secret.json", headers=headers)
+        assert response.status_code == 403
+        response = await ds.client.get(
+            "/data/-/query.json?sql=select+*+from+secret", headers=headers
+        )
+        assert response.status_code == 403
+        # Distinct values let us detect if an operation targets secret
+        # instead of the newly created secret\n table.
+        response = await ds.client.post(
+            "/data/-/create",
+            json={"table": "secret\n", "row": {"id": 1, "value": "decoy"}, "pk": "id"},
+            headers=headers,
+        )
+        assert response.status_code == 201, response.text
+        # ~0A is Datasette's URL encoding for the line feed in the table name.
+        if operation in ("read", "read_row"):
+            # Both table and row endpoints must return only the permitted row.
+            path = "/1.json" if operation == "read_row" else ".json"
+            response = await ds.client.get(
+                "/data/secret~0A" + path + "?_shape=array", headers=headers
+            )
+            assert response.status_code == 200, response.text
+            assert response.json() == [{"id": 1, "value": "decoy"}]
+        else:
+            # Renaming must move the permitted table, preserving its contents
+            # and removing its old name from the database.
+            response = await ds.client.post(
+                "/data/secret~0A/-/alter",
+                json={"operations": [{"op": "rename_table", "args": {"to": "moved"}}]},
+                headers=headers,
+            )
+            assert response.status_code == 200, response.text
+            db = ds.get_database("data")
+            assert (
+                await db.execute('select value from "moved"')
+            ).single_value() == "decoy"
+            assert "secret\n" not in await db.table_names()
+        # Verify that the protected table and its data are unchanged, and that
+        # the API still denies access to it.
+        db = ds.get_database("data")
+        assert (
+            await db.execute('select value from "secret"')
+        ).single_value() == "private"
+        response = await ds.client.get("/data/secret.json", headers=headers)
+        assert response.status_code == 403
+    finally:
+        ds.close()
+
+
 def _insert_and_fetch_created(conn, table, insert_sql):
     cursor = conn.execute(insert_sql)
     return conn.execute(
-        "select created, typeof(created) from {} where rowid = ?".format(
-            escape_sqlite(table)
-        ),
+        f"select created, typeof(created) from {escape_sqlite(table)} where rowid = ?",
         (cursor.lastrowid,),
     ).fetchone()
 
@@ -66,6 +167,82 @@ BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"}
 BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}'
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_fallback", (False, True))
+@pytest.mark.parametrize(
+    "operation", ("insert", "upsert", "update", "delete", "create", "create_uppercase")
+)
+@pytest.mark.parametrize(
+    "module,definition,values,shadow_suffix",
+    (
+        ("fts5", "body", "'original'", "_content"),
+        ("fts4", "body", "'original'", "_content"),
+        ("rtree", "id, minx, maxx", "1, 0, 1", "_rowid"),
+    ),
+)
+@pytest.mark.parametrize("shadow", (False, True))
+async def test_structured_writes_require_ordinary_tables(
+    ds_write,
+    monkeypatch,
+    use_fallback,
+    operation,
+    module,
+    definition,
+    values,
+    shadow_suffix,
+    shadow,
+):
+    if use_fallback:
+        monkeypatch.setattr("datasette.utils.sqlite.supports_table_list", lambda: False)
+    db = ds_write.get_database("data")
+    await db.execute_write(f"create virtual table indexed using {module}({definition})")
+    await db.execute_write(f"insert into indexed values ({values})")
+    table = "indexed" + (shadow_suffix if shadow else "")
+    row = (await db.execute(f"select rowid, * from {escape_sqlite(table)}")).dicts()[0]
+    pks = await db.primary_keys(table)
+    pk_value = row[pks[0] if pks else "rowid"]
+    before = await db.execute_fn(lambda conn: list(conn.iterdump()))
+
+    if operation in ("create", "create_uppercase"):
+        path = "/data/-/create"
+        body = {
+            "table": table.upper() if operation == "create_uppercase" else table,
+            "rows": [row],
+        }
+    elif operation in ("update", "delete"):
+        path = f"/data/{table}/{pk_value}/-/{operation}"
+        body = {"update": row} if operation == "update" else {}
+    else:
+        path = f"/data/{table}/-/{operation}"
+        body = {"rows": [row]}
+    response = await ds_write.client.post(
+        path, json=body, headers=_headers(write_token(ds_write))
+    )
+    assert response.status_code == 400, response.text
+    assert response.json()["errors"] == ["Structured writes require an ordinary table"]
+    assert await db.execute_fn(lambda conn: list(conn.iterdump())) == before
+
+
+@pytest.mark.asyncio
+async def test_structured_writes_to_content_table_maintain_fts(ds_write):
+    db = ds_write.get_database("data")
+    await db.execute_write_fn(
+        lambda conn: sqlite_utils.Database(conn)["docs"].enable_fts(
+            ["title"], create_triggers=True
+        )
+    )
+    response = await ds_write.client.post(
+        "/data/docs/-/insert",
+        json={"row": {"id": 1, "title": "ordinary content"}},
+        headers=_headers(write_token(ds_write)),
+    )
+    assert response.status_code == 201, response.text
+    matches = await db.execute(
+        "select rowid from docs_fts where docs_fts match ?", ["ordinary"]
+    )
+    assert [row[0] for row in matches.rows] == [1]
+
+
 @pytest.mark.asyncio
 async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write):
     token = write_token(ds_write)
@@ -241,7 +418,7 @@ async def test_insert_row(ds_write, content_type):
         "/data/docs/-/insert",
         json={"row": {"title": "Test", "score": 1.2, "age": 5}},
         headers={
-            "Authorization": "Bearer {}".format(token),
+            "Authorization": f"Bearer {token}",
             "Content-Type": content_type,
         },
     )
@@ -286,11 +463,7 @@ async def test_insert_row_alter(ds_write):
 @pytest.mark.parametrize("return_rows", (True, False))
 async def test_insert_rows(ds_write, return_rows):
     token = write_token(ds_write)
-    data = {
-        "rows": [
-            {"title": "Test {}".format(i), "score": 1.0, "age": 5} for i in range(20)
-        ]
-    }
+    data = {"rows": [{"title": f"Test {i}", "score": 1.0, "age": 5} for i in range(20)]}
     if return_rows:
         data["return"] = True
     response = await ds_write.client.post(
@@ -314,8 +487,7 @@ async def test_insert_rows(ds_write, return_rows):
     ).dicts()
     assert len(actual_rows) == 20
     assert actual_rows == [
-        {"id": i + 1, "title": "Test {}".format(i), "score": 1.0, "age": 5}
-        for i in range(20)
+        {"id": i + 1, "title": f"Test {i}", "score": 1.0, "age": 5} for i in range(20)
     ]
     assert response.json()["ok"] is True
     if return_rows:
@@ -561,13 +733,13 @@ async def test_insert_or_upsert_row_errors(
         )
     if special_case == "bad_token":
         token += "bad"
-    kwargs = dict(
-        json=input,
-        headers={
-            "Authorization": "Bearer {}".format(token),
+    kwargs = {
+        "json": input,
+        "headers": {
+            "Authorization": f"Bearer {token}",
             "Content-Type": "application/json",
         },
-    )
+    }
 
     if special_case != "bad_token":
         actor_response = (
@@ -622,7 +794,7 @@ async def test_upsert_permissions_per_table(ds_write, allowed):
         "/data/docs/-/upsert",
         json={"rows": [{"id": 1, "title": "One"}]},
         headers={
-            "Authorization": "Bearer {}".format(token),
+            "Authorization": f"Bearer {token}",
         },
     )
     if allowed:
@@ -859,9 +1031,7 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path):
     # Should be a single row
     assert (
         await ds_write.client.get(
-            "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(
-                table
-            )
+            f"/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{table}"
         )
     ).json() == [1]
     # Now delete the row
@@ -869,14 +1039,12 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path):
         # Special case for that rowid table
         delete_path = (
             await ds_write.client.get(
-                "/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{}".format(
-                    table
-                )
+                f"/data/-/query.json?_shape=arrayfirst&sql=select+rowid+from+{table}"
             )
         ).json()[0]
 
     delete_response = await ds_write.client.post(
-        "/data/{}/{}/-/delete".format(table, delete_path),
+        f"/data/{table}/{delete_path}/-/delete",
         headers=_headers(write_token(ds_write)),
     )
     assert delete_response.status_code == 200
@@ -889,9 +1057,7 @@ async def test_delete_row(ds_write, table, row_for_create, pks, delete_path):
     assert event.pks == str(delete_path).split(",")
     assert (
         await ds_write.client.get(
-            "/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{}".format(
-                table
-            )
+            f"/data/-/query.json?_shape=arrayfirst&sql=select+count(*)+from+{table}"
         )
     ).json() == [0]
 
@@ -941,7 +1107,7 @@ async def test_update_row_invalid_key(ds_write):
 
     pk = await _insert_row(ds_write)
 
-    path = "/data/docs/{}/-/update".format(pk)
+    path = f"/data/docs/{pk}/-/update"
     response = await ds_write.client.post(
         path,
         json={"update": {"title": "New title"}, "bad_key": 1},
@@ -960,7 +1126,7 @@ async def test_update_row_invalid_key(ds_write):
 async def test_update_row_alter(ds_write):
     token = write_token(ds_write, permissions=["ur", "at"])
     pk = await _insert_row(ds_write)
-    path = "/data/docs/{}/-/update".format(pk)
+    path = f"/data/docs/{pk}/-/update"
     response = await ds_write.client.post(
         path,
         json={"update": {"title": "New title", "extra": "extra"}, "alter": True},
@@ -1116,9 +1282,9 @@ async def test_alter_table_integer_default_expr(
     assert expected_schema in data["schema"]
 
     columns = await db.execute("select * from pragma_table_info('docs')")
-    created_column = [
+    created_column = next(
         column for column in columns.dicts() if column["name"] == "created"
-    ][0]
+    )
     assert created_column["type"] == "INTEGER"
     assert expected_schema in created_column["dflt_value"]
 
@@ -1305,7 +1471,7 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w
 
 @pytest.mark.asyncio
 async def test_foreign_key_suggestions(ds_write):
-    token = write_token(ds_write, permissions=["at"])
+    token = write_token(ds_write, permissions=["alter-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
     await db.execute_write("insert into owners (id) values (1), (2), (3)")
@@ -1371,7 +1537,7 @@ async def test_foreign_key_suggestions_permission_denied(ds_write):
 
 @pytest.mark.asyncio
 async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch):
-    token = write_token(ds_write, permissions=["at"])
+    token = write_token(ds_write, permissions=["alter-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
 
@@ -1402,7 +1568,7 @@ async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch):
 
 @pytest.mark.asyncio
 async def test_foreign_key_targets(ds_write):
-    token = write_token(ds_write, permissions=["ct"])
+    token = write_token(ds_write, permissions=["create-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
     await db.execute_write("create table categories (slug varchar(30) primary key)")
@@ -1419,7 +1585,8 @@ async def test_foreign_key_targets(ds_write):
     await db.execute_write("create table no_pk (name text)")
     try:
         await db.execute_write("create virtual table search_docs using fts5(body)")
-    except Exception:
+    except Exception:  # noqa: BLE001, S110
+        # FTS5 is not available in every SQLite build
         pass
 
     response = await ds_write.client.get(
@@ -1625,7 +1792,7 @@ async def test_update_row(ds_write, input, expected_errors, use_return):
     token = write_token(ds_write)
     pk = await _insert_row(ds_write)
 
-    path = "/data/docs/{}/-/update".format(pk)
+    path = f"/data/docs/{pk}/-/update"
 
     data = {"update": input}
     if use_return:
@@ -1660,7 +1827,7 @@ async def test_update_row(ds_write, input, expected_errors, use_return):
 
     # And fetch the row to check it's updated
     response = await ds_write.client.get(
-        "/data/docs/{}.json?_shape=array".format(pk),
+        f"/data/docs/{pk}.json?_shape=array",
     )
     assert response.status_code == 200
     row = response.json()[0]
@@ -1734,6 +1901,42 @@ async def test_drop_table(ds_write, scenario):
         assert (await ds_write.client.get("/data/docs")).status_code == 404
 
 
+@pytest.mark.asyncio
+async def test_drop_table_cleans_up_fts(ds_write):
+    db = ds_write.get_database("data")
+
+    def enable_fts(conn):
+        sqlite_utils.Database(conn)["docs"].enable_fts(["title"], create_triggers=True)
+
+    await db.execute_write_fn(enable_fts)
+    assert {
+        row[0]
+        for row in await db.execute(
+            "select name from sqlite_master where type = 'table' and name like 'docs_fts%'"
+        )
+    } == {
+        "docs_fts",
+        "docs_fts_config",
+        "docs_fts_data",
+        "docs_fts_docsize",
+        "docs_fts_idx",
+    }
+
+    response = await ds_write.client.post(
+        "/data/docs/-/drop",
+        json={"confirm": True},
+        headers=_headers(write_token(ds_write)),
+    )
+
+    assert response.json() == {"ok": True}
+    assert [
+        row[0]
+        for row in await db.execute(
+            "select name from sqlite_master where type = 'table' and name like 'docs_fts%'"
+        )
+    ] == []
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "input,expected_status,expected_response,expected_events",
@@ -2306,7 +2509,7 @@ async def test_create_table_integer_default_expr(
     ds_write, default_expr, minimum_value, expected_schema
 ):
     token = write_token(ds_write)
-    table = "default_{}".format(default_expr)
+    table = f"default_{default_expr}"
     response = await ds_write.client.post(
         "/data/-/create",
         json={
@@ -2334,7 +2537,7 @@ async def test_create_table_integer_default_expr(
 
     row = await db.execute_write_fn(
         lambda conn: _insert_and_fetch_created(
-            conn, table, "insert into {} default values".format(escape_sqlite(table))
+            conn, table, f"insert into {escape_sqlite(table)} default values"
         )
     )
     assert row[0] > minimum_value
@@ -2717,3 +2920,119 @@ async def test_create_using_alter_against_existing_table(
         insert_rows_event = ds_write._tracked_events[1]
         assert insert_rows_event.name == "insert-rows"
         assert insert_rows_event.num_rows == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("denied_action", "request_body"),
+    (
+        (
+            "insert-row",
+            {
+                "table": "salaries",
+                "rows": [{"id": 9, "note": "INJ-VIA-CREATE"}],
+            },
+        ),
+        (
+            "update-row",
+            {
+                "table": "salaries",
+                "rows": [{"id": 1, "note": "REPLACED"}],
+                "pk": "id",
+                "replace": True,
+            },
+        ),
+        (
+            "alter-table",
+            {
+                "table": "salaries",
+                "rows": [{"id": 9, "note": "INSERTED", "extra": "NEW"}],
+                "alter": True,
+            },
+        ),
+    ),
+)
+async def test_create_table_existing_table_respects_table_level_denial(
+    denied_action, request_body
+):
+    # GHSA-53fc-rhfg-h7qp issue 2: POST /db/-/create against an existing table
+    # inserts rows into it, so insert-row (and update-row / alter-table) must be
+    # checked against the TableResource, not just the DatabaseResource.
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                # id=editor user has each permission at the database level, but
+                # the selected action is explicitly denied on the salaries table
+                "data": {
+                    "permissions": {
+                        "create-table": {"id": "editor"},
+                        "insert-row": {"id": "editor"},
+                        "update-row": {"id": "editor"},
+                        "alter-table": {"id": "editor"},
+                    },
+                    "tables": {
+                        "salaries": {"permissions": {denied_action: False}},
+                    },
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database(
+        f"create_table_existing_table_denied_{denied_action}", name="data"
+    )
+    await db.execute_write("create table salaries (id integer primary key, note text)")
+    await db.execute_write("insert into salaries values (1, 'TOPSECRET-A')")
+    await ds.invoke_startup()
+
+    if denied_action == "insert-row":
+        # Sanity: direct insert into salaries is denied for this actor
+        direct = await ds.client.post(
+            "/data/salaries/-/insert",
+            actor={"id": "editor"},
+            json={"row": {"id": 9, "note": "INJ-DIRECT"}},
+        )
+        assert direct.status_code == 403
+
+    response = await ds.client.post(
+        "/data/-/create",
+        actor={"id": "editor"},
+        json=request_body,
+    )
+    assert response.status_code == 403, response.json()
+    assert response.json()["errors"] == [f"Permission denied: need {denied_action}"]
+    rows = (await db.execute("select id, note from salaries order by id")).rows
+    assert [tuple(r) for r in rows] == [(1, "TOPSECRET-A")]
+    assert await db.table_columns("salaries") == ["id", "note"]
+
+
+@pytest.mark.asyncio
+async def test_create_table_respects_predeclared_table_level_denial():
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                "data": {
+                    "permissions": {
+                        "create-table": {"id": "editor"},
+                        "insert-row": {"id": "editor"},
+                    },
+                    "tables": {
+                        "planned_table": {"permissions": {"insert-row": False}},
+                    },
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database("create_table_predeclared_denial", name="data")
+    await ds.invoke_startup()
+
+    response = await ds.client.post(
+        "/data/-/create",
+        actor={"id": "editor"},
+        json={"table": "planned_table", "rows": [{"id": 1}]},
+    )
+
+    assert response.status_code == 403, response.json()
+    assert response.json()["errors"] == ["Permission denied: need insert-row"]
+    assert not await db.table_exists("planned_table")

tests/test_auth.py

@@ -1,14 +1,18 @@
+import time
+from unittest.mock import AsyncMock
+
+import pytest
 from bs4 import BeautifulSoup as Soup
-from .utils import cookie_was_deleted, last_event
 from click.testing import CliRunner
-from datasette.utils import baseconv
+
 from datasette.cli import cli
 from datasette.resources import (
     DatabaseResource,
     TableResource,
 )
-import pytest
-import time
+from datasette.utils import baseconv
+
+from .utils import cookie_was_deleted, last_event
 
 
 @pytest.mark.asyncio
@@ -204,7 +208,7 @@ def test_auth_create_token(
     assert response2.status == 200
     if errors:
         for error in errors:
-            assert '<p class="message-error">{}</p>'.format(error) in response2.text
+            assert f'<p class="message-error">{error}</p>' in response2.text
     else:
         # Check create-token event
         event = last_event(app_client.ds)
@@ -228,12 +232,41 @@ def test_auth_create_token(
         # And test that token
         response3 = app_client.get(
             "/-/actor.json",
-            headers={"Authorization": "Bearer {}".format("dstok_{}".format(token))},
+            headers={"Authorization": "Bearer {}".format(f"dstok_{token}")},
         )
         assert response3.status == 200
         assert response3.json["actor"]["id"] == "test"
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["GET", "POST"])
+@pytest.mark.parametrize(
+    "restrictions",
+    [
+        {},
+        {"a": ["vi"]},
+        {"d": {"db": ["vd"]}},
+        {"r": {"db": {"t1": ["vt"]}}},
+    ],
+    ids=["empty", "instance", "database", "table"],
+)
+async def test_auth_create_token_not_allowed_for_restricted_actors(
+    bare_ds, monkeypatch, method, restrictions
+):
+    create_token = AsyncMock()
+    monkeypatch.setattr(bare_ds, "create_token", create_token)
+
+    response = await bare_ds.client.request(
+        method,
+        "/-/create-token",
+        actor={"id": "test", "_r": restrictions},
+    )
+
+    assert response.status_code == 403
+    assert "Restricted actors cannot create API tokens" in response.text
+    create_token.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_auth_create_token_not_allowed_for_tokens(ds_client):
     ds_tok = ds_client.ds.sign(
@@ -241,7 +274,7 @@ async def test_auth_create_token_not_allowed_for_tokens(ds_client):
     )
     response = await ds_client.get(
         "/-/create-token",
-        headers={"Authorization": "Bearer dstok_{}".format(ds_tok)},
+        headers={"Authorization": f"Bearer dstok_{ds_tok}"},
     )
     assert response.status_code == 403
 
@@ -286,12 +319,12 @@ async def test_auth_with_dstok_token(ds_client, scenario, should_work):
     elif scenario == "invalid_token":
         token = "invalid"
     if token:
-        token = "dstok_{}".format(token)
+        token = f"dstok_{token}"
     if scenario == "allow_signed_tokens_off":
         ds_client.ds._settings["allow_signed_tokens"] = False
     headers = {}
     if token:
-        headers["Authorization"] = "Bearer {}".format(token)
+        headers["Authorization"] = f"Bearer {token}"
     response = await ds_client.get("/-/actor.json", headers=headers)
     try:
         if should_work:
@@ -338,7 +371,7 @@ def test_cli_create_token(app_client, expires):
     assert details.keys() == expected_keys
     assert details["a"] == "test"
     response = app_client.get(
-        "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)}
+        "/-/actor.json", headers={"Authorization": f"Bearer {token}"}
     )
     if expires is None or expires > 0:
         expected_actor = {
@@ -521,3 +554,25 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client):
         )
         is not True
     ), "Root without root_enabled should not automatically get set-column-type"
+
+
+@pytest.mark.parametrize("expire_after", (1, 300, 3600, 30 * 24 * 60 * 60))
+def test_set_actor_cookie_honours_expire_after(expire_after):
+    # GHSA-53fc-rhfg-h7qp issue 4: expire_after is documented as a number of
+    # seconds, but every value was being replaced with 24 hours.
+    from datasette.app import Datasette
+    from datasette.utils.asgi import Response
+
+    ds = Datasette(memory=True)
+    response = Response.text("")
+    before = int(time.time())
+    ds.set_actor_cookie(response, {"id": "test"}, expire_after=expire_after)
+    after = int(time.time())
+
+    (header,) = response._set_cookie_headers
+    assert header.startswith("ds_actor=")
+    value = header[len("ds_actor=") :].split(";", 1)[0]
+    data = ds.unsign(value, "actor")
+    assert data["a"] == {"id": "test"}
+    expires_at = baseconv.base62.decode(data["e"])
+    assert before + expire_after <= expires_at <= after + expire_after

tests/test_background_tasks.py

@@ -0,0 +1,381 @@
+"""
+Tests for datasette.add_background_task() / start_background_tasks() and the
+BackgroundTask / BackgroundTaskSupervisor machinery in
+datasette/background_tasks.py.
+"""
+
+import asyncio
+import contextlib
+import logging
+
+import httpx2
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.plugins import pm
+
+
+async def _drive_lifespan_startup(app):
+    """Send a single lifespan.startup message into app's ASGI lifespan loop
+    and return the list of messages sent back, without ever sending
+    lifespan.shutdown. Copied from tests/test_lifespan.py's helper of the
+    same name - mirrors what a real server does: after startup completes
+    it parks waiting for the next event, and we cancel that wait once
+    we've observed the startup response.
+    """
+    messages_sent = []
+    startup_responded = asyncio.Event()
+    delivered = False
+
+    async def receive():
+        nonlocal delivered
+        if not delivered:
+            delivered = True
+            return {"type": "lifespan.startup"}
+        await asyncio.Event().wait()
+
+    async def send(message):
+        messages_sent.append(message)
+        startup_responded.set()
+
+    task = asyncio.create_task(app({"type": "lifespan"}, receive, send))
+    try:
+        await asyncio.wait_for(startup_responded.wait(), timeout=5)
+    finally:
+        task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await task
+    return messages_sent
+
+
+@pytest.mark.asyncio
+async def test_tasks_registered_in_startup_hook_run_after_lifespan_startup():
+    # Two tasks registered by one plugin's startup hook - order preserved,
+    # both running after lifespan startup completes, and no HTTP request
+    # of any kind is issued anywhere in this test.
+    events = []
+
+    async def task_one(datasette):
+        events.append("task_one")
+        # Wait indefinitely to simulate long-lived background work, keeping
+        # the task "running" for the assertions below until cleanup cancels it.
+        await asyncio.Event().wait()
+
+    async def task_two(datasette):
+        events.append("task_two")
+        await asyncio.Event().wait()
+
+    class TwoTaskPlugin:
+        __name__ = "TwoTaskPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(task_one, name="task-one")
+                datasette.add_background_task(task_two, name="task-two")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(TwoTaskPlugin(), name="two_task_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan_startup(app)
+        assert {"type": "lifespan.startup.complete"} in messages
+
+        handles = ds._background_tasks.tasks()
+        assert [h.name for h in handles] == ["task-one", "task-two"]
+
+        # Let both tasks run their first line of code.
+        await asyncio.sleep(0)
+        assert handles[0].state == "running"
+        assert handles[1].state == "running"
+        assert events == ["task_one", "task_two"]
+    finally:
+        pm.unregister(name="two_task_plugin")
+        await ds._background_tasks.cancel_all(grace=1.0)
+
+
+@pytest.mark.asyncio
+async def test_launch_waits_for_every_startup_hook_before_running_any_task():
+    # PluginA registers a task from its startup hook; PluginB does the
+    # same from ITS startup hook, which runs after PluginA's (forced with
+    # tryfirst=True on A). Even though A's registration happens first,
+    # A's task body must not actually execute until every startup hook -
+    # including B's - has finished, since launch only happens after
+    # invoke_startup() completes. This is the ordering guarantee that
+    # dissolves datasette-cron's tryfirst=True launch hack.
+    hook_call_order = []
+    seen_names_when_a_ran = {}
+
+    async def task_a(datasette):
+        seen_names_when_a_ran["names"] = [
+            h.name for h in datasette._background_tasks.tasks()
+        ]
+
+    async def task_b(datasette):
+        pass
+
+    class PluginA:
+        __name__ = "PluginA"
+
+        @hookimpl(tryfirst=True)
+        def startup(self, datasette):
+            async def inner():
+                hook_call_order.append("A")
+                datasette.add_background_task(task_a, name="task-a")
+
+            return inner
+
+    class PluginB:
+        __name__ = "PluginB"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                hook_call_order.append("B")
+                datasette.add_background_task(task_b, name="task-b")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(PluginA(), name="plugin_a")
+    pm.register(PluginB(), name="plugin_b")
+    try:
+        await ds.start_background_tasks()
+        # Confirm A's startup hook really did run (and register task-a)
+        # strictly before B's startup hook ran.
+        assert hook_call_order == ["A", "B"]
+
+        handles = ds._background_tasks.tasks()
+        await asyncio.wait_for(asyncio.gather(*[h.task for h in handles]), timeout=5)
+        # Yet by the time task-a's own body executed (after launch, which
+        # only happens once every startup hook - including B's - has
+        # finished), task-b was already registered.
+        assert "task-b" in seen_names_when_a_ran["names"]
+    finally:
+        pm.unregister(name="plugin_a")
+        pm.unregister(name="plugin_b")
+
+
+@pytest.mark.asyncio
+async def test_concurrent_first_requests_launch_background_tasks_exactly_once():
+    launch_count = {"n": 0}
+
+    async def counting_task(datasette):
+        launch_count["n"] += 1
+
+    class CountingTaskPlugin:
+        __name__ = "CountingTaskPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(counting_task, name="counting-task")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(CountingTaskPlugin(), name="counting_task_plugin")
+    try:
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            responses = await asyncio.gather(
+                *[client.get("/-/versions.json") for _ in range(10)]
+            )
+        assert all(response.status_code == 200 for response in responses)
+
+        handles = ds._background_tasks.tasks()
+        assert len(handles) == 1
+        await asyncio.wait_for(handles[0].task, timeout=5)
+        assert launch_count["n"] == 1
+    finally:
+        pm.unregister(name="counting_task_plugin")
+
+
+@pytest.mark.asyncio
+async def test_post_launch_registration_starts_immediately_and_cancel_works():
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()  # nothing registered yet, but launched
+
+    started = asyncio.Event()
+
+    async def long_running(datasette):
+        started.set()
+        await asyncio.Event().wait()
+
+    handle = ds.add_background_task(long_running, name="dynamic-task")
+    # Registered after launch: starts immediately rather than sitting in
+    # "registered" limbo.
+    assert handle.state == "running"
+    assert handle.task is not None
+
+    await asyncio.wait_for(started.wait(), timeout=5)
+    assert handle.state == "running"
+
+    handle.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await handle.task
+    await asyncio.sleep(0)
+    assert handle.state == "cancelled"
+
+
+@pytest.mark.asyncio
+async def test_pre_launch_registration_starts_as_registered():
+    ds = Datasette(memory=True)
+
+    async def task(datasette):
+        pass
+
+    handle = ds.add_background_task(task, name="buffered-task")
+    assert handle.state == "registered"
+    assert handle.task is None
+
+    handle.cancel()  # not yet launched: deregisters instead of cancelling
+    assert handle not in ds._background_tasks.tasks()
+
+
+@pytest.mark.asyncio
+async def test_crashing_task_logs_traceback_and_state_is_crashed(caplog):
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()
+
+    survivor_ran = asyncio.Event()
+
+    async def crashing_task(datasette):
+        raise RuntimeError("kaboom")
+
+    async def survivor(datasette):
+        survivor_ran.set()
+
+    with caplog.at_level(logging.ERROR, logger="datasette.background_tasks"):
+        crash_handle = ds.add_background_task(crashing_task, name="crashing_task")
+        survivor_handle = ds.add_background_task(survivor, name="survivor")
+        await asyncio.wait_for(
+            asyncio.gather(
+                crash_handle.task, survivor_handle.task, return_exceptions=True
+            ),
+            timeout=5,
+        )
+
+    assert crash_handle.state == "crashed"
+    assert isinstance(crash_handle.exception, RuntimeError)
+    assert str(crash_handle.exception) == "kaboom"
+
+    # The crash must not affect any other task.
+    assert survivor_ran.is_set()
+    assert survivor_handle.state == "completed"
+
+    assert "crashing_task" in caplog.text
+    assert "kaboom" in caplog.text
+    assert "Traceback" in caplog.text
+    assert "RuntimeError" in caplog.text
+
+
+def test_name_collisions_get_suffixed_and_explicit_names_are_respected():
+    ds = Datasette(memory=True)
+
+    async def noop(datasette):
+        pass
+
+    async def another_noop(datasette):
+        pass
+
+    h1 = ds.add_background_task(noop, name="dup")
+    h2 = ds.add_background_task(another_noop, name="dup")
+    h3 = ds.add_background_task(noop, name="dup")
+    assert [h1.name, h2.name, h3.name] == ["dup", "dup-2", "dup-3"]
+
+    h_explicit = ds.add_background_task(noop, name="explicit-name")
+    assert h_explicit.name == "explicit-name"
+
+    h_default = ds.add_background_task(noop)
+    assert h_default.name == noop.__qualname__
+
+
+@pytest.mark.asyncio
+async def test_start_background_tasks_on_bare_datasette():
+    # The headless-CLI path (datasette-rss's `fetch --due` shape): no
+    # server, no lifespan, no first HTTP request - just an explicit call.
+    ran = asyncio.Event()
+
+    async def task(datasette):
+        ran.set()
+
+    ds = Datasette([])
+    assert ds._startup_invoked is False
+
+    handle = ds.add_background_task(task, name="headless-task")
+    assert handle.state == "registered"
+
+    await ds.start_background_tasks()
+
+    assert ds._startup_invoked is True
+    await asyncio.wait_for(ran.wait(), timeout=5)
+    await asyncio.wait_for(handle.task, timeout=5)
+    # handle.task being done only guarantees the coroutine has returned,
+    # not that our done-callback (which updates handle.state) has run yet -
+    # asyncio schedules done-callbacks via call_soon, and awaiting an
+    # already-done future/task returns immediately without giving the loop
+    # a chance to drain its ready queue. Yield once to let it run.
+    await asyncio.sleep(0)
+    assert handle.state == "completed"
+
+
+@pytest.mark.asyncio
+async def test_cancel_all_cancels_running_tasks_and_leaves_completed_alone():
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()
+
+    async def forever(datasette):
+        await asyncio.Event().wait()
+
+    async def quick(datasette):
+        return "done"
+
+    forever_handle = ds.add_background_task(forever, name="forever")
+    quick_handle = ds.add_background_task(quick, name="quick")
+    await asyncio.wait_for(quick_handle.task, timeout=5)
+    assert quick_handle.state == "completed"
+
+    await ds._background_tasks.cancel_all(grace=1.0)
+
+    assert forever_handle.state == "cancelled"
+    assert quick_handle.state == "completed"
+
+
+@pytest.mark.asyncio
+async def test_cancel_all_logs_stragglers_that_outlive_the_grace_period(caplog):
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()
+
+    async def stubborn(datasette):
+        with contextlib.suppress(asyncio.CancelledError):
+            await asyncio.sleep(10)
+        # Swallowing CancelledError above and returning normally simulates
+        # a task that ignores cancellation for longer than the grace period.
+        await asyncio.sleep(10)
+
+    handle = ds.add_background_task(stubborn, name="stubborn-task")
+    # Let the task actually start running and reach its first sleep (inside
+    # the CancelledError-suppressing block) before cancelling it - a task
+    # cancelled before it has ever run its first step never enters that
+    # block at all (the throw happens before the coroutine body starts),
+    # so it would finish cancelling immediately instead of behaving like a
+    # straggler.
+    await asyncio.sleep(0)
+
+    with caplog.at_level(logging.WARNING, logger="datasette.background_tasks"):
+        await ds._background_tasks.cancel_all(grace=0.1)
+
+    assert "stubborn-task" in caplog.text
+
+    # Clean up: actually cancel it now that the test has made its
+    # assertion, so it doesn't leak past the end of the test.
+    handle.task.cancel()
+    with contextlib.suppress(asyncio.CancelledError):
+        await handle.task

tests/test_base_view.py

@@ -1,8 +1,10 @@
-from datasette.views.base import View
+import json
+
+import pytest
+
 from datasette import Request, Response
 from datasette.app import Datasette
-import json
-import pytest
+from datasette.views.base import View
 
 
 class GetView(View):

tests/test_cli.py

@@ -1,23 +1,28 @@
-from .fixtures import (
-    make_app_client,
-    TestClient as _TestClient,
-    EXPECTED_PLUGINS,
-)
-from datasette.app import SETTINGS
-from datasette.plugins import DEFAULT_PLUGINS, pm
-from datasette.cli import cli, serve
-from datasette.version import __version__
-from datasette.utils import tilde_encode
-from datasette.utils.sqlite import sqlite3
-from click.testing import CliRunner
 import io
 import json
 import pathlib
-import pytest
 import sys
 import textwrap
 from unittest import mock
 
+import pytest
+from click.testing import CliRunner
+
+from datasette.app import SETTINGS
+from datasette.cli import cli, serve
+from datasette.plugins import DEFAULT_PLUGINS, pm
+from datasette.utils import tilde_encode
+from datasette.utils.sqlite import sqlite3
+from datasette.version import __version__
+
+from .fixtures import (
+    EXPECTED_PLUGINS,
+    make_app_client,
+)
+from .fixtures import (
+    TestClient as _TestClient,
+)
+
 
 def test_inspect_cli(app_client):
     runner = CliRunner()
@@ -460,7 +465,7 @@ def test_serve_create(tmpdir):
 @pytest.mark.parametrize("argument", ("-c", "--config"))
 @pytest.mark.parametrize("format_", ("json", "yaml"))
 def test_serve_config(tmpdir, argument, format_):
-    config_path = tmpdir / "datasette.{}".format(format_)
+    config_path = tmpdir / f"datasette.{format_}"
     config_path.write_text(
         (
             "settings:\n  default_page_size: 5\n"
@@ -513,13 +518,13 @@ def test_weird_database_names(tmpdir, filename):
     result1 = runner.invoke(cli, [db_path, "--get", "/"])
     assert result1.exit_code == 0, result1.output
     filename_no_stem = filename.rsplit(".", 1)[0]
-    expected_link = '<a href="/{}">{}</a>'.format(
-        tilde_encode(filename_no_stem), filename_no_stem
+    expected_link = (
+        f'<a href="/{tilde_encode(filename_no_stem)}">{filename_no_stem}</a>'
     )
     assert expected_link in result1.output
     # Now try hitting that database page
     result2 = runner.invoke(
-        cli, [db_path, "--get", "/{}".format(tilde_encode(filename_no_stem))]
+        cli, [db_path, "--get", f"/{tilde_encode(filename_no_stem)}"]
     )
     assert result2.exit_code == 0, result2.output
 

tests/test_cli_serve_get.py

@@ -1,8 +1,10 @@
+import json
+import textwrap
+
+from click.testing import CliRunner
+
 from datasette.cli import cli
 from datasette.plugins import pm
-from click.testing import CliRunner
-import textwrap
-import json
 
 
 def test_serve_with_get(tmp_path_factory):
@@ -44,9 +46,59 @@ def test_serve_with_get(tmp_path_factory):
 
     # Annoyingly that new test plugin stays resident - we need
     # to manually unregister it to avoid conflict with other tests
-    to_unregister = [
+    to_unregister = next(
         p for p in pm.get_plugins() if p.__name__ == "init_for_serve_with_get.py"
-    ][0]
+    )
+    pm.unregister(to_unregister)
+
+
+def test_serve_with_get_does_not_launch_background_tasks(tmp_path_factory):
+    # --get must never launch background tasks, even though its TestClient
+    # request
+    # flows through the full ASGI stack (including the AsgiRunOnFirstRequest
+    # fallback that would otherwise launch them). The plugin's startup hook
+    # itself still runs (registration happens) - only the launch is
+    # suppressed, so the sentinel file the background task would write must
+    # never appear.
+    plugins_dir = tmp_path_factory.mktemp("plugins_for_get_background_tasks")
+    sentinel = plugins_dir / "sentinel.txt"
+    (plugins_dir / "bg_task_for_get.py").write_text(
+        textwrap.dedent(
+            f"""
+        from datasette import hookimpl
+
+        @hookimpl
+        def startup(datasette):
+            async def inner():
+                async def task(datasette):
+                    with open("{sentinel!s}", "w") as fp:
+                        fp.write("ran")
+
+                datasette.add_background_task(task, name="get-sentinel-task")
+
+            return inner
+    """,
+        ),
+        "utf-8",
+    )
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        [
+            "serve",
+            "--memory",
+            "--plugins-dir",
+            str(plugins_dir),
+            "--get",
+            "/_memory/-/query.json?sql=select+1",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert not sentinel.exists()
+
+    to_unregister = next(
+        p for p in pm.get_plugins() if p.__name__ == "bg_task_for_get.py"
+    )
     pm.unregister(to_unregister)
 
 

tests/test_cli_serve_server.py

@@ -1,11 +1,15 @@
-import httpx
-import pytest
+import signal
 import socket
+import subprocess
+import time
+
+import httpx2
+import pytest
 
 
 @pytest.mark.serial
 def test_serve_localhost_http(ds_localhost_http_server):
-    response = httpx.get("http://localhost:8041/_memory.json")
+    response = httpx2.get("http://localhost:8041/_memory.json")
     assert {
         "database": "_memory",
         "path": "/_memory",
@@ -19,11 +23,205 @@ def test_serve_localhost_http(ds_localhost_http_server):
 )
 def test_serve_unix_domain_socket(ds_unix_domain_socket_server):
     _, uds = ds_unix_domain_socket_server
-    transport = httpx.HTTPTransport(uds=uds)
-    client = httpx.Client(transport=transport)
-    response = client.get("http://localhost/_memory.json")
+    transport = httpx2.HTTPTransport(uds=uds)
+    with httpx2.Client(transport=transport) as client:
+        response = client.get("http://localhost/_memory.json")
     assert {
         "database": "_memory",
         "path": "/_memory",
         "tables": [],
     }.items() <= response.json().items()
+
+
+# Shaped after datasette-litestream's startup hook, which schedules a
+# background task with asyncio.get_running_loop().create_task(...):
+# https://github.com/datasette/datasette-litestream
+MARKER_TASK_PLUGIN = """
+import asyncio
+from datasette import hookimpl
+from datasette.utils.asgi import Response
+
+
+@hookimpl
+def startup(datasette):
+    datasette._startup_calls = getattr(datasette, "_startup_calls", 0) + 1
+
+    async def _mark():
+        # Must await before setting the flag: a task with no internal
+        # await point could finish on the throwaway loop before it
+        # closed, masking the regression this test guards against.
+        await asyncio.sleep(0.2)
+        datasette._marker_task_ran = True
+
+    asyncio.get_running_loop().create_task(_mark())
+
+
+@hookimpl
+def register_routes():
+    async def marker_status(datasette):
+        return Response.json(
+            {
+                "marker_task_ran": getattr(datasette, "_marker_task_ran", False),
+                "startup_calls": getattr(datasette, "_startup_calls", 0),
+            }
+        )
+
+    return [(r"^/-/marker-task-ran$", marker_status)]
+"""
+
+
+STARTUP_ERROR_PLUGIN = """
+from datasette import hookimpl
+from datasette.utils import StartupError
+
+
+@hookimpl
+def startup(datasette):
+    raise StartupError("boom from plugin")
+"""
+
+
+@pytest.mark.serial
+def test_startup_hook_background_task_runs_on_serving_loop(serve_with_plugins):
+    """
+    Litestream-shaped regression test: a startup hook that does
+    asyncio.get_running_loop().create_task(...) must have that task
+    actually execute before/while the server is handling requests. This
+    only holds if invoke_startup() and uvicorn.Server.serve() share one
+    event loop. This test fails against unmodified main, where
+    invoke_startup() runs on a throwaway loop that is closed before
+    uvicorn opens its own loop to serve.
+    """
+    _, port = serve_with_plugins({"marker_task_plugin": MARKER_TASK_PLUGIN})
+    # The fixture has already waited for the server to answer requests. The
+    # marker task deliberately awaits before setting its flag, so poll for a
+    # moment rather than assuming it landed before the first request arrived.
+    deadline = time.time() + 3.0
+    payload = {}
+    while time.time() < deadline:
+        payload = httpx2.get(
+            f"http://127.0.0.1:{port}/-/marker-task-ran", timeout=1.0
+        ).json()
+        if payload["marker_task_ran"]:
+            break
+        time.sleep(0.05)
+    assert payload.get("marker_task_ran"), (
+        "The startup hook's asyncio.create_task(...) never ran - "
+        "invoke_startup() and the server are not sharing an event loop"
+    )
+    # Polling above means this test would also pass if the startup hook were
+    # re-run on the serving loop by the first-request fallback - which would
+    # hide exactly the bug being tested. invoke_startup() is idempotent today
+    # so that cannot happen; assert it explicitly so that if the idempotency
+    # guard is ever removed this test fails loudly instead of silently
+    # becoming a no-op.
+    assert payload["startup_calls"] == 1, (
+        "startup hook ran {} times - the marker may have been set by a "
+        "re-run on the serving loop rather than by the original task".format(
+            payload["startup_calls"]
+        )
+    )
+
+
+@pytest.mark.serial
+def test_startup_error_fails_fast_before_port_binds(serve_with_plugins):
+    """
+    A "startup" plugin hook that raises StartupError must fail fast: print
+    the message, exit non-zero, and never accept a connection on the port -
+    the failure must happen before uvicorn.Server binds the socket.
+    """
+    proc, port = serve_with_plugins(
+        {"startup_error_plugin": STARTUP_ERROR_PLUGIN}, wait_for_startup=False
+    )
+    stdout, _ = proc.communicate(timeout=15)
+    output = stdout.decode("utf-8")
+    assert proc.returncode not in (0, None), output
+    assert "boom from plugin" in output, output
+
+    # Nothing is listening on the port now the process has exited. This
+    # confirms the socket was not left bound; on its own it cannot prove the
+    # failure preceded the bind, since a port nothing ever touched also
+    # refuses connections.
+    with (
+        pytest.raises(OSError),
+        socket.create_connection(("127.0.0.1", port), timeout=0.2),
+    ):
+        pass
+
+
+# Verify that SIGTERM and SIGINT sent to `datasette serve` trigger uvicorn's
+# lifespan.shutdown event and run the plugin shutdown hooks. The plugin below
+# writes a sentinel file from its shutdown hook so the tests can check that
+# cleanup ran after the server subprocess exits.
+SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE = """
+import pathlib
+from datasette import hookimpl
+
+SENTINEL_PATH = {sentinel_path!r}
+
+
+@hookimpl
+def shutdown(datasette):
+    pathlib.Path(SENTINEL_PATH).write_text("shutdown ran", "utf-8")
+"""
+
+
+def _start_serve_with_shutdown_sentinel(serve_with_plugins, tmp_path):
+    sentinel_path = tmp_path / "shutdown-sentinel.txt"
+    proc, _ = serve_with_plugins(
+        {
+            "shutdown_sentinel_plugin": SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE.format(
+                sentinel_path=str(sentinel_path)
+            )
+        }
+    )
+    return proc, sentinel_path
+
+
+@pytest.mark.serial
+def test_sigterm_runs_shutdown_hooks(serve_with_plugins, tmp_path):
+    ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel(
+        serve_with_plugins, tmp_path
+    )
+    assert not sentinel_path.exists()
+    ds_proc.send_signal(signal.SIGTERM)
+    try:
+        ds_proc.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        ds_proc.kill()
+        ds_proc.wait()
+        raise AssertionError(
+            "datasette serve did not exit within 10s of SIGTERM\n"
+            + ds_proc.stdout.read().decode("utf-8")
+        )
+    output = ds_proc.stdout.read().decode("utf-8")
+    assert sentinel_path.exists(), (
+        "shutdown hook never wrote its sentinel file after SIGTERM\n" + output
+    )
+    assert sentinel_path.read_text("utf-8") == "shutdown ran"
+
+
+@pytest.mark.serial
+@pytest.mark.skipif(
+    not hasattr(signal, "SIGINT"), reason="Requires signal.SIGINT support"
+)
+def test_sigint_runs_shutdown_hooks(serve_with_plugins, tmp_path):
+    ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel(
+        serve_with_plugins, tmp_path
+    )
+    assert not sentinel_path.exists()
+    ds_proc.send_signal(signal.SIGINT)
+    try:
+        ds_proc.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        ds_proc.kill()
+        ds_proc.wait()
+        raise AssertionError(
+            "datasette serve did not exit within 10s of SIGINT\n"
+            + ds_proc.stdout.read().decode("utf-8")
+        )
+    output = ds_proc.stdout.read().decode("utf-8")
+    assert sentinel_path.exists(), (
+        "shutdown hook never wrote its sentinel file after SIGINT\n" + output
+    )
+    assert sentinel_path.read_text("utf-8") == "shutdown ran"

tests/test_column_types.py

@@ -1,7 +1,11 @@
 import json
 import logging
+import time
 
+import markupsafe
+import pytest
 from bs4 import BeautifulSoup as Soup
+
 from datasette.app import Datasette
 from datasette.column_types import (
     ColumnType,
@@ -9,11 +13,7 @@ from datasette.column_types import (
 )
 from datasette.hookspecs import hookimpl
 from datasette.plugins import pm
-from datasette.utils import error_body, sqlite3
-from datasette.utils import StartupError
-import markupsafe
-import pytest
-import time
+from datasette.utils import StartupError, error_body, sqlite3
 
 
 @pytest.fixture
@@ -31,6 +31,7 @@ def ds_ct(tmp_path_factory):
         "'https://example.com', '{\"key\": \"value\"}')"
     )
     db.commit()
+    db.close()
     ds = Datasette(
         [db_path],
         config={
@@ -70,6 +71,7 @@ def ds_ct_editor_permission(tmp_path_factory):
         "'https://example.com', '{\"key\": \"value\"}')"
     )
     db.commit()
+    db.close()
     ds = Datasette(
         [db_path],
         config={
@@ -104,7 +106,7 @@ def write_token(ds, actor_id="root", permissions=None):
 
 def _headers(token):
     return {
-        "Authorization": "Bearer {}".format(token),
+        "Authorization": f"Bearer {token}",
         "Content-Type": "application/json",
     }
 
@@ -568,44 +570,6 @@ async def test_url_render_cell(ds_ct):
     assert "https://example.com" in rendered["website"]
 
 
-@pytest.mark.asyncio
-async def test_url_render_cell_truncates(tmp_path_factory):
-    # truncate_cells_html should also truncate the displayed text of url
-    # column type links, while keeping the full URL in the href - refs #1805
-    db_directory = tmp_path_factory.mktemp("dbs")
-    db_path = str(db_directory / "data.db")
-    db = sqlite3.connect(db_path)
-    db.execute("create table posts (id integer primary key, website text)")
-    long_url = (
-        "https://images.openfoodfacts.org/images/products/000/000/000/088/"
-        "nutrition_fr.5.200.jpg"
-    )
-    db.execute("insert into posts values (1, ?)", [long_url])
-    db.commit()
-    db.close()
-    ds = Datasette(
-        [db_path],
-        settings={"truncate_cells_html": 30},
-        config={
-            "databases": {
-                "data": {"tables": {"posts": {"column_types": {"website": "url"}}}}
-            }
-        },
-    )
-    await ds.invoke_startup()
-    response = await ds.client.get("/data/posts.json?_extra=render_cell")
-    assert response.status_code == 200
-    rendered = response.json()["render_cell"][0]["website"]
-    # The full URL is preserved in the href
-    assert f'href="{long_url}"' in rendered
-    # ...but the visible link text is truncated
-    soup = Soup(rendered, "html.parser")
-    link_text = soup.find("a").text
-    assert link_text != long_url
-    assert "…" in link_text
-    ds.close()
-
-
 @pytest.mark.asyncio
 async def test_email_render_cell(ds_ct):
     await ds_ct.invoke_startup()
@@ -950,6 +914,90 @@ async def test_html_table_page_rendering(ds_ct):
     assert 'href="https://example.com"' in html
 
 
+# --- url column type link text truncation, refs #1805 ---
+
+LONG_URL = (
+    "https://images.openfoodfacts.org/images/products/000/000/000/088/"
+    "nutrition_fr.5.200.jpg"
+)
+
+
+@pytest.fixture
+def ds_long_url(tmp_path_factory):
+    db_path = str(tmp_path_factory.mktemp("dbs") / "data.db")
+    db = sqlite3.connect(db_path)
+    db.execute("create table posts (id integer primary key, website text)")
+    db.execute("insert into posts values (1, ?)", [LONG_URL])
+    db.execute("insert into posts values (2, 'not a url')")
+    db.commit()
+    db.close()
+    ds = Datasette(
+        [db_path],
+        settings={"truncate_cells_html": 30},
+        config={
+            "databases": {
+                "data": {"tables": {"posts": {"column_types": {"website": "url"}}}}
+            }
+        },
+    )
+    yield ds
+    ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "truncate_cells_html,expected_text",
+    (
+        (30, "https://images.openfoodfac….jpg"),
+        # 0 disables truncation
+        (0, LONG_URL),
+    ),
+)
+async def test_url_column_type_truncates_link_text(
+    ds_long_url, truncate_cells_html, expected_text
+):
+    # The table page truncates the link text but the href keeps the full URL
+    ds_long_url._settings["truncate_cells_html"] = truncate_cells_html
+    await ds_long_url.invoke_startup()
+    response = await ds_long_url.client.get("/data/posts")
+    assert response.status_code == 200
+    link = Soup(response.text, "html.parser").select_one("td.col-website a")
+    assert link["href"] == LONG_URL
+    assert link.text == expected_text
+
+
+@pytest.mark.asyncio
+async def test_url_column_type_full_link_text_on_row_page(ds_long_url):
+    # Row pages show full values, so the link text is not truncated there
+    await ds_long_url.invoke_startup()
+    response = await ds_long_url.client.get("/data/posts/1")
+    assert response.status_code == 200
+    link = Soup(response.text, "html.parser").select_one("td.col-website a")
+    assert link["href"] == LONG_URL
+    assert link.text == LONG_URL
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("path", ("/data/posts.json", "/data/posts/1.json"))
+async def test_url_column_type_full_link_text_in_json(ds_long_url, path):
+    # truncate_cells_html only applies to the HTML table view
+    await ds_long_url.invoke_startup()
+    response = await ds_long_url.client.get(path + "?_extra=render_cell")
+    assert response.status_code == 200
+    rendered = response.json()["render_cell"][0]["website"]
+    assert rendered == f'<a href="{LONG_URL}">{LONG_URL}</a>'
+
+
+@pytest.mark.asyncio
+async def test_url_column_type_non_http_value_unchanged(ds_long_url):
+    await ds_long_url.invoke_startup()
+    response = await ds_long_url.client.get("/data/posts")
+    assert response.status_code == 200
+    td = Soup(response.text, "html.parser").select("td.col-website")[1]
+    assert td.find("a") is None
+    assert td.text == "not a url"
+
+
 @pytest.mark.asyncio
 async def test_set_column_type_ui_data_hidden_without_permission(ds_ct):
     await ds_ct.invoke_startup()

tests/test_config_dir.py

@@ -1,10 +1,12 @@
 import json
 import pathlib
+
 import pytest
 
 from datasette.app import Datasette
-from datasette.utils.sqlite import sqlite3
 from datasette.utils import StartupError
+from datasette.utils.sqlite import sqlite3
+
 from .fixtures import TestClient as _TestClient
 
 PLUGIN = """
@@ -109,7 +111,7 @@ def test_settings(config_dir_client):
 def test_plugins(config_dir_client):
     response = config_dir_client.get("/-/plugins.json")
     assert 200 == response.status
-    plugins = response.json["plugins"]
+    plugins = response.json
     assert "hooray.py" in {p["name"] for p in plugins}
     assert "non_py_file.txt" not in {p["name"] for p in plugins}
     assert "mypy_cache" not in {p["name"] for p in plugins}

tests/test_crossdb.py

@@ -1,7 +1,9 @@
-from datasette.cli import cli
-from click.testing import CliRunner
-import urllib
 import sqlite3
+import urllib
+
+from click.testing import CliRunner
+
+from datasette.cli import cli
 
 
 def test_crossdb_join(app_client_two_attached_databases_crossdb_enabled):
@@ -40,7 +42,7 @@ def test_crossdb_warning_if_too_many_databases(tmp_path_factory):
     db_dir = tmp_path_factory.mktemp("dbs")
     dbs = []
     for i in range(11):
-        path = str(db_dir / "db_{}.db".format(i))
+        path = str(db_dir / f"db_{i}.db")
         conn = sqlite3.connect(path)
         conn.execute("vacuum")
         conn.close()

tests/test_csrf_middleware.py

@@ -44,7 +44,7 @@ async def _run_middleware(scope):
     await mw(scope, None, send)
     if inner_called:
         return ("allowed",)
-    start = [m for m in sent if m["type"] == "http.response.start"][0]
+    start = next(m for m in sent if m["type"] == "http.response.start")
     return ("blocked", start["status"])
 
 

tests/test_csv.py

@@ -1,8 +1,10 @@
-from datasette.app import Datasette
-from bs4 import BeautifulSoup as Soup
-import pytest
 import urllib.parse
 
+import pytest
+from bs4 import BeautifulSoup as Soup
+
+from datasette.app import Datasette
+
 EXPECTED_TABLE_CSV = """id,content
 1,hello
 2,world
@@ -164,6 +166,66 @@ async def test_custom_sql_csv(ds_client):
     assert response.text == EXPECTED_CUSTOM_CSV
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("download", (False, True))
+@pytest.mark.parametrize(
+    "query_string,expected_error",
+    (
+        ("sql=select+blah", "no such column: blah"),
+        ("sql=select+*+from+missing", "no such table: missing"),
+        ("sql=select+from", 'near "from": syntax error'),
+        (
+            "sql=delete+from+simple_primary_key",
+            "Statement must be a SELECT",
+        ),
+        ("", "?sql= is required"),
+        (
+            "sql=select+sleep(0.01)&_timelimit=5",
+            (
+                "SQL query took too long. The time limit is"
+                " controlled by the sql_time_limit_ms setting."
+            ),
+        ),
+    ),
+)
+async def test_custom_sql_csv_errors(ds_client, query_string, expected_error, download):
+    if download:
+        query_string += "&_dl=1"
+    response = await ds_client.get(f"/fixtures/-/query.csv?{query_string}")
+    assert response.status_code == 400
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert "content-disposition" not in response.headers
+    assert response.text == expected_error
+
+
+@pytest.mark.asyncio
+async def test_custom_sql_csv_error_head(ds_client):
+    response = await ds_client.head("/fixtures/-/query.csv?sql=select+blah")
+    assert response.status_code == 400
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.content == b""
+
+
+@pytest.mark.asyncio
+async def test_custom_sql_csv_error_cors():
+    ds = Datasette(cors=True)
+    response = await ds.client.get("/_memory/-/query.csv?sql=select+blah")
+    assert response.status_code == 400
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.headers["access-control-allow-origin"] == "*"
+    assert response.text == "no such column: blah"
+
+
+@pytest.mark.asyncio
+async def test_table_csv_error(ds_client):
+    response = await ds_client.get(
+        "/fixtures/simple_primary_key.csv?_where=blah&_stream=1"
+    )
+    assert response.status_code == 400
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.text == "no such column: blah"
+
+
 @pytest.mark.asyncio
 async def test_table_csv_download(ds_client):
     response = await ds_client.get("/fixtures/simple_primary_key.csv?_dl=1")
@@ -227,6 +289,20 @@ async def test_table_csv_stream(ds_client):
     assert len([b for b in response.content.split(b"\r\n") if b]) == 1002
 
 
+@pytest.mark.asyncio
+async def test_view_csv_stream(ds_client):
+    # Without _stream should return header + 100 rows:
+    response = await ds_client.get("/fixtures/paginated_view.csv?_size=max")
+    assert len([b for b in response.content.split(b"\r\n") if b]) == 101
+    # With _stream=1 should paginate through all pages and return header + 202 rows
+    response = await ds_client.get("/fixtures/paginated_view.csv?_stream=1")
+    lines = [b for b in response.content.split(b"\r\n") if b]
+    assert len(lines) == 203
+    # Ensure there are no duplicate rows from looping
+    assert len(set(lines[1:])) == 202
+    assert lines[0] == b"content,content_extra"
+
+
 def test_csv_trace(app_client_with_trace):
     response = app_client_with_trace.get("/fixtures/simple_primary_key.csv?_trace=1")
     assert response.headers["content-type"] == "text/html; charset=utf-8"

tests/test_custom_pages.py

@@ -1,5 +1,7 @@
 import pathlib
+
 import pytest
+
 from .fixtures import make_app_client
 
 TEST_TEMPLATE_DIRS = str(pathlib.Path(__file__).parent / "test_templates")

tests/test_default_deny.py

@@ -1,4 +1,5 @@
 import pytest
+
 from datasette.app import Datasette
 from datasette.resources import DatabaseResource, TableResource
 

tests/test_docs.py

@@ -2,20 +2,22 @@
 Tests to ensure certain things are documented.
 """
 
-from datasette import app, utils
+import re
+from pathlib import Path
+
+import pytest
+
 import datasette.fixtures  # noqa: F401
+from datasette import app, utils
 from datasette.app import Datasette
 from datasette.filters import Filters
-from pathlib import Path
-import pytest
-import re
 
 docs_path = Path(__file__).parent.parent / "docs"
 label_re = re.compile(r"\.\. _([^\s:]+):")
 
 
 def get_headings(content, underline="-"):
-    heading_re = re.compile(r"(\w+)(\([^)]*\))?\n\{}+\n".format(underline))
+    heading_re = re.compile(rf"(\w+)(\([^)]*\))?\n\{underline}+\n")
     return {h[0] for h in heading_re.findall(content)}
 
 
@@ -25,14 +27,20 @@ def get_labels(filename):
 
 
 @pytest.fixture(scope="session")
-def settings_headings():
-    return get_headings((docs_path / "settings.rst").read_text(), "~")
+def settings_sections():
+    content = (docs_path / "settings.rst").read_text()
+    sections = re.split(r"^(\w+)\n~+\n", content, flags=re.MULTILINE)
+    return dict(zip(sections[1::2], sections[2::2]))
 
 
-def test_settings_are_documented(settings_headings, subtests):
+def test_settings_are_documented(settings_sections, subtests):
     for setting in app.SETTINGS:
         with subtests.test(setting=setting.name):
-            assert setting.name in settings_headings
+            assert setting.name in settings_sections
+            assert (
+                f'setting_default(cog, "{setting.name}")'
+                in settings_sections[setting.name]
+            )
 
 
 @pytest.fixture(scope="session")

tests/test_docs_plugins.py

@@ -1,10 +1,11 @@
 # fmt: off
 # -- start datasette_with_plugin_fixture --
-from datasette import hookimpl
-from datasette.app import Datasette
 import pytest
 import pytest_asyncio
 
+from datasette import hookimpl
+from datasette.app import Datasette
+
 
 @pytest_asyncio.fixture
 async def datasette_with_plugin():

tests/test_error_shape.py

@@ -17,8 +17,10 @@ present and the legacy "title" key must not be.
 https://github.com/simonw/datasette/issues - 1.0 API consistency
 """
 
-import pytest
 import time
+
+import pytest
+
 from datasette.app import Datasette
 from datasette.utils import sqlite3
 
@@ -86,7 +88,7 @@ async def test_write_api_validation_error_shape(ds_error_shape):
         "/data/docs/-/insert",
         json={"rows": [{"nope": 1}, {"also_nope": 2}]},
         headers={
-            "Authorization": "Bearer {}".format(token),
+            "Authorization": f"Bearer {token}",
             "Content-Type": "application/json",
         },
     )
@@ -410,7 +412,7 @@ async def test_expired_token_returns_401(ds_error_shape):
         )
     )
     response = await ds_error_shape.client.get(
-        "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)}
+        "/-/actor.json", headers={"Authorization": f"Bearer {token}"}
     )
     data = assert_canonical_error(response, 401)
     assert "expired" in data["error"].lower()
@@ -446,7 +448,7 @@ async def test_valid_token_still_authenticates(ds_error_shape):
         )
     )
     response = await ds_error_shape.client.get(
-        "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)}
+        "/-/actor.json", headers={"Authorization": f"Bearer {token}"}
     )
     assert response.status_code == 200
     assert response.json()["actor"]["id"] == "root"
@@ -477,7 +479,7 @@ async def test_token_when_signed_tokens_disabled_returns_401(tmp_path_factory):
             ds.sign({"a": "root", "t": int(time.time())}, namespace="token")
         )
         response = await ds.client.get(
-            "/-/actor.json", headers={"Authorization": "Bearer {}".format(token)}
+            "/-/actor.json", headers={"Authorization": f"Bearer {token}"}
         )
         data = assert_canonical_error(response, 401)
         assert "not enabled" in data["error"]
@@ -642,7 +644,7 @@ async def test_query_list_size_rejects_non_integer(ds_client):
 @pytest.mark.asyncio
 @pytest.mark.parametrize("endpoint", ("allowed", "rules"))
 async def test_debug_endpoints_use_size_and_page_parameters(ds_error_shape, endpoint):
-    base = "/-/{}.json?action=view-instance".format(endpoint)
+    base = f"/-/{endpoint}.json?action=view-instance"
     ok = await ds_error_shape.client.get(
         base + "&_size=1&_page=1", actor={"id": "root"}
     )

tests/test_extras.py

@@ -1,4 +1,5 @@
 import asyncio
+from typing import ClassVar
 
 import pytest
 
@@ -7,7 +8,7 @@ from datasette.extras import Extra, ExtraRegistry, ExtraScope
 
 class SlowValueExtra(Extra):
     description = "Returns context['value'], optionally slowly"
-    scopes = {ExtraScope.TABLE}
+    scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE}
 
     async def resolve(self, context):
         if context["slow"]:
@@ -17,7 +18,7 @@ class SlowValueExtra(Extra):
 
 class DependentExtra(Extra):
     description = "Depends on slow_value"
-    scopes = {ExtraScope.TABLE}
+    scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE}
 
     async def resolve(self, context, slow_value):
         return slow_value + 1
@@ -25,7 +26,7 @@ class DependentExtra(Extra):
 
 class InternalOnlyExtra(Extra):
     description = "Internal extra for HTML templates only"
-    scopes = {ExtraScope.TABLE}
+    scopes: ClassVar[set[ExtraScope]] = {ExtraScope.TABLE}
     public = False
 
     async def resolve(self, context):
@@ -52,7 +53,7 @@ def _registered_extra_classes():
 @pytest.mark.parametrize("cls", _registered_extra_classes(), ids=lambda cls: cls.key())
 def test_registered_extras_have_descriptions(cls):
     # Every registered extra is part of the documented template/JSON contract
-    assert cls.description, "{} is missing a description".format(cls.__name__)
+    assert cls.description, f"{cls.__name__} is missing a description"
 
 
 def test_registry_is_built_once_per_scope():

tests/test_facets.py

@@ -1,11 +1,49 @@
+import json
+from urllib.parse import parse_qsl, urlsplit
+
+import pytest
+
 from datasette.app import Datasette
 from datasette.database import Database
-from datasette.facets import Facet, ColumnFacet, ArrayFacet, DateFacet
-from datasette.utils.asgi import Request
+from datasette.facets import (
+    ArrayFacet,
+    ColumnFacet,
+    DateFacet,
+    Facet,
+    load_facet_configs,
+)
 from datasette.utils import detect_json1
+from datasette.utils.asgi import Request
+
 from .fixtures import make_app_client
-import json
-import pytest
+
+
+@pytest.mark.parametrize(
+    "query_string",
+    ("_facets=ignored", "_facet=state&_facets=ignored", "_facets=ignored&_facet=state"),
+)
+@pytest.mark.parametrize("table_config", ({}, {"facets": ["state"]}))
+def test_facet_configs_ignore_unrelated_prefixes(query_string, table_config):
+    expected = load_facet_configs(
+        Request.fake("/?_facet=state" if "_facet=" in query_string else "/"),
+        table_config,
+    )
+    assert (
+        load_facet_configs(Request.fake("/?" + query_string), table_config) == expected
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "query_string",
+    ("_facet=state&_facets=ignored", "_facets=ignored&_facet=state"),
+)
+async def test_facet_ignores_unrelated_prefixes(ds_client, query_string):
+    response = await ds_client.get("/fixtures/facetable.json?" + query_string)
+    assert response.status_code == 200
+    facets = response.json()["facet_results"]["results"]
+    assert set(facets) == {"state"}
+    assert facets["state"]["results"]
 
 
 @pytest.mark.asyncio
@@ -148,6 +186,63 @@ async def test_column_facet_results(ds_client):
     ] == buckets
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "column,filters,remaining",
+    [
+        ("COUNTY", "COUNTY=Lee", []),
+        ("COUNTY", "COUNTY__exact=Lee", []),
+        ("COUNTY", "COUNTY=Lee&COUNTY__exact=Lee", []),
+        ("COUNTY", "COUNTY__exact=Lee&COUNTY__exact=Lee", []),
+        (
+            "COUNTY",
+            "COUNTY__exact=Lee&COUNTY__exact=Polk",
+            [("COUNTY__exact", "Polk")],
+        ),
+        ("_county", "_county__exact=Lee", []),
+        ("_county", "_county__exact=Lee&_county=Lee", [("_county", "Lee")]),
+    ],
+)
+async def test_column_facet_selected_exact_filters(
+    ds_client, column, filters, remaining
+):
+    facet = ColumnFacet(
+        ds_client.ds,
+        Request.fake(f"/?_facet={column}&{filters}&other=keep&_sort={column}"),
+        database="fixtures",
+        sql=f"select 'Lee' as {column}",
+    )
+    buckets, timed_out = await facet.facet_results()
+    assert not timed_out
+    result = buckets[0]["results"][0]
+    assert result["selected"] is True
+    assert parse_qsl(urlsplit(result["toggle_url"]).query) == [
+        ("_facet", column),
+        *remaining,
+        ("other", "keep"),
+        ("_sort", column),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_column_facet_underscore_argument_is_not_a_filter(ds_client):
+    facet = ColumnFacet(
+        ds_client.ds,
+        Request.fake("/?_facet=_county&_county=Lee"),
+        database="fixtures",
+        sql="select 'Lee' as _county",
+    )
+    buckets, timed_out = await facet.facet_results()
+    assert not timed_out
+    result = buckets[0]["results"][0]
+    assert result["selected"] is False
+    assert parse_qsl(urlsplit(result["toggle_url"]).query) == [
+        ("_facet", "_county"),
+        ("_county", "Lee"),
+        ("_county__exact", "Lee"),
+    ]
+
+
 @pytest.mark.asyncio
 async def test_column_facet_results_column_starts_with_underscore(ds_client):
     facet = ColumnFacet(
@@ -537,7 +632,7 @@ async def test_facet_size():
         for j in range(1, 4):
             await db.execute_write(
                 "insert into neighbourhoods (city, neighbourhood) values (?, ?)",
-                ["City {}".format(i), "Neighbourhood {}".format(j)],
+                [f"City {i}", f"Neighbourhood {j}"],
             )
     response = await ds.client.get(
         "/test_facet_size/neighbourhoods.json?_extra=suggested_facets"

tests/test_filters.py

@@ -1,7 +1,8 @@
-from datasette.filters import Filters, through_filters, where_filters, search_filters
-from datasette.utils.asgi import Request
 import pytest
 
+from datasette.filters import Filters, search_filters, through_filters, where_filters
+from datasette.utils.asgi import Request
+
 
 @pytest.mark.parametrize(
     "args,expected_where,expected_params",
@@ -65,12 +66,12 @@ import pytest
         # JSON arraycontains, arraynotcontains
         (
             (("Availability+Info__arraycontains", "yes"),),
-            [":p0 in (select value from json_each([table].[Availability+Info]))"],
+            [':p0 in (select value from json_each("table"."Availability+Info"))'],
             ["yes"],
         ),
         (
             (("Availability+Info__arraynotcontains", "yes"),),
-            [":p0 not in (select value from json_each([table].[Availability+Info]))"],
+            [':p0 not in (select value from json_each("table"."Availability+Info"))'],
             ["yes"],
         ),
     ],
@@ -82,6 +83,35 @@ def test_build_where(args, expected_where, expected_params):
     assert {f"p{i}": param for i, param in enumerate(expected_params)} == actual_params
 
 
+@pytest.mark.parametrize(
+    "key,expected_where",
+    (
+        (
+            'has"quote__exact',
+            '"has""quote" = :p0',
+        ),
+        (
+            'has"quote__isnull',
+            '"has""quote" is null',
+        ),
+        (
+            "has]bracket__arraycontains",
+            ':p0 in (select value from json_each("table"."has]bracket"))',
+        ),
+    ),
+)
+def test_build_where_escapes_column_names(key, expected_where):
+    filters = Filters(((key, "value"),))
+    sql_bits, _ = filters.build_where_clauses("table")
+    assert sql_bits == [expected_where]
+
+
+def test_build_where_escapes_table_name():
+    filters = Filters((("tags__arraycontains", "value"),))
+    sql_bits, _ = filters.build_where_clauses("items]bracket")
+    assert sql_bits == [':p0 in (select value from json_each("items]bracket"."tags"))']
+
+
 @pytest.mark.asyncio
 async def test_through_filters_from_request(ds_client):
     request = Request.fake(

tests/test_fts_permissions.py

@@ -0,0 +1,412 @@
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.permissions import Action, PermissionSQL, _permission_check_cache
+from datasette.resources import DatabaseResource, TableResource
+from datasette.utils.sqlite import sqlite3, sqlite_derived_table_dependencies
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ["fts4", "fts5"])
+@pytest.mark.parametrize("actor", [None, {"id": "root"}], ids=["anonymous", "root"])
+async def test_derived_permissions_allow_one_hop_but_deny_nested_sources(
+    fts_module, actor
+):
+    class InspectPlugin:
+        @hookimpl
+        def register_actions(self):
+            return [
+                Action(
+                    name="inspect-derived",
+                    description="Inspect a table",
+                    resource_class=TableResource,
+                    also_requires="view-table",
+                )
+            ]
+
+        @hookimpl
+        def permission_resources_sql(self, action):
+            if action == "inspect-derived":
+                return PermissionSQL(
+                    sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'inspect allowed' AS reason"
+                )
+
+    ds = Datasette(memory=True)
+    ds.pm.register(InspectPlugin(), name="inspect-derived-test")
+    db = ds.add_memory_database(
+        f"derived_one_hop_{fts_module}_{actor is not None}", name="data"
+    )
+    await db.execute_write("create table Documents (body text)")
+    await db.execute_write(
+        f"create virtual table Search using {fts_module}(body, content='Documents')"
+    )
+    await db.execute_write(
+        f"create virtual table Nested using {fts_module}(body, content='sEaRcH')"
+    )
+    await ds.invoke_startup()
+    token = _permission_check_cache.set({})
+    try:
+        # Both direct permissions are allowed, but a derived source makes its
+        # dependent unavailable even to an actor who can view the whole chain.
+        # Check and cache Search first so its cached grant cannot grant Nested.
+        for table, expected in (
+            ("Documents", True),
+            ("Search", True),
+            ("Nested", False),
+            ("Search_docsize", False),
+        ):
+            for spelling in (table, table.upper(), table.lower()):
+                assert await ds.allowed_many(
+                    actions=["view-table", "inspect-derived"],
+                    resource=TableResource("data", spelling),
+                    actor=actor,
+                ) == {"view-table": expected, "inspect-derived": expected}
+
+        page = await ds.allowed_resources(
+            "view-table", actor, parent="data", include_is_private=True, limit=1000
+        )
+        allowed = {resource.child for resource in page.resources}
+        assert {"Documents", "Search"}.issubset(allowed)
+        assert "Nested" not in allowed
+        assert "Search_docsize" not in allowed
+    finally:
+        _permission_check_cache.reset(token)
+        ds.pm.unregister(name="inspect-derived-test")
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("listing", [False, True], ids=["individual", "listing"])
+async def test_derived_permission_discovery_error_is_retried(monkeypatch, listing):
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(f"derived_discovery_error_{listing}", name="data")
+    await db.execute_write("create table documents (id integer primary key)")
+    await ds.invoke_startup()
+
+    class UnavailableSchema:
+        def execute(self, sql):
+            raise sqlite3.DatabaseError("schema temporarily unavailable")
+
+    async def check():
+        if listing:
+            return await ds.allowed_resources("view-table", parent="data")
+        return await ds.allowed(
+            action="view-table", resource=TableResource("data", "documents")
+        )
+
+    token = _permission_check_cache.set({})
+    try:
+        with monkeypatch.context() as patch:
+            patch.setattr(
+                "datasette.database.sqlite_derived_table_dependencies",
+                lambda conn: sqlite_derived_table_dependencies(UnavailableSchema()),
+            )
+            with pytest.raises(sqlite3.DatabaseError, match="schema temporarily"):
+                await check()
+
+        # Failed discovery must not cache an empty map or a permission grant.
+        assert db._cached_derived_table_dependencies is None
+        assert not _permission_check_cache.get()
+        result = await check()
+        if listing:
+            assert [resource.child for resource in result.resources] == ["documents"]
+        else:
+            assert result is True
+        assert db._cached_derived_table_dependencies is not None
+    finally:
+        _permission_check_cache.reset(token)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ("fts4", "fts5"))
+async def test_external_content_fts_inherits_content_table_view_permission(fts_module):
+    actor = {"id": "reader"}
+    secret_marker = "ISSUE_17_EXTERNAL_CONTENT_FTS_SECRET"
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": {"id": "reader"},
+                "view-database": {"id": "reader"},
+                "view-table": {"id": "reader"},
+                "execute-sql": {"id": "nobody"},
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": False}},
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database(f"issue_17_{fts_module}_permissions", name="data")
+    await db.execute_write("create table secret (id integer primary key, body text)")
+    await db.execute_write(
+        "insert into secret (body) values (?)",
+        [secret_marker],
+    )
+    fts_options = "body, content='secret'"
+    if fts_module == "fts5":
+        fts_options += ", content_rowid='id'"
+    await db.execute_write(
+        f"create virtual table secret_fts using {fts_module}({fts_options})"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        assert "secret_fts" in await db.hidden_table_names()
+        assert (
+            await ds.allowed(
+                action="execute-sql",
+                resource=DatabaseResource("data"),
+                actor=actor,
+            )
+            is False
+        )
+
+        direct = await ds.client.get("/data/secret.json", actor=actor)
+        assert direct.status_code == 403
+
+        companion = await ds.client.get(
+            "/data/secret_fts.json?_shape=array",
+            actor=actor,
+        )
+        assert companion.status_code in (403, 404), (
+            "An automatically hidden external-content FTS table must inherit "
+            "the content table's view denial or be unavailable: "
+            f"{companion.text}"
+        )
+        assert secret_marker not in companion.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ("fts4", "fts5"))
+@pytest.mark.parametrize("contentless", (False, True), ids=("internal", "contentless"))
+async def test_fts_shadow_tables_inherit_logical_table_view_permission(
+    fts_module, contentless
+):
+    table_config = {
+        "secret_fts": {"permissions": {"view-table": False}},
+        # An explicit allow on one implementation table must not override
+        # the logical FTS table's denial.
+        "secret_fts_docsize": {"permissions": {"view-table": True}},
+    }
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+                "view-table": True,
+                "execute-sql": False,
+            },
+            "databases": {"data": {"tables": table_config}},
+        },
+    )
+    db = ds.add_memory_database(
+        f"issue_17_{fts_module}_{'contentless' if contentless else 'internal'}",
+        name="data",
+    )
+    options = "body, content=''" if contentless else "body"
+    await db.execute_write(
+        f"create virtual table secret_fts using {fts_module}({options})"
+    )
+    await db.execute_write(
+        "insert into secret_fts(rowid, body) values (1, 'ISSUE_17_SHADOW_SECRET')"
+    )
+    await ds.invoke_startup()
+
+    try:
+        dependencies = await db.derived_table_dependencies()
+        shadow_tables = sorted(
+            table for table, source in dependencies.items() if source == "secret_fts"
+        )
+        assert shadow_tables
+        assert "secret_fts_docsize" in shadow_tables
+
+        for shadow_table in shadow_tables:
+            assert (
+                await ds.allowed(
+                    action="view-table",
+                    resource=TableResource("data", shadow_table),
+                )
+                is False
+            )
+            response = await ds.client.get(f"/data/{shadow_table}.json?_shape=array")
+            assert response.status_code == 403
+            assert "ISSUE_17_SHADOW_SECRET" not in response.text
+
+        allowed = await ds.allowed_resources("view-table", parent="data", limit=1000)
+        allowed_names = {resource.child for resource in allowed.resources}
+        assert not set(shadow_tables).intersection(allowed_names)
+
+        database_json = await ds.client.get("/data.json")
+        assert database_json.status_code == 200
+        for shadow_table in shadow_tables:
+            assert shadow_table not in database_json.text
+
+        schema_json = await ds.client.get("/data/-/schema.json")
+        assert schema_json.status_code == 200
+        for shadow_table in shadow_tables:
+            assert shadow_table not in schema_json.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "content_allowed,companion_allowed,expected",
+    (
+        (False, True, False),
+        (True, False, False),
+        (True, True, True),
+    ),
+)
+async def test_external_content_and_companion_permissions_are_both_required(
+    content_allowed, companion_allowed, expected
+):
+    ds = Datasette(
+        memory=True,
+        default_deny=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": content_allowed}},
+                        "secret_fts": {
+                            "permissions": {"view-table": companion_allowed}
+                        },
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database(
+        f"issue_17_explicit_{int(content_allowed)}_{int(companion_allowed)}",
+        name="data",
+    )
+    await db.execute_write("create table secret(id integer primary key, body text)")
+    await db.execute_write("insert into secret(body) values ('ISSUE_17_MATRIX_SECRET')")
+    await db.execute_write(
+        "create virtual table secret_fts using fts5("
+        "body, content='secret', content_rowid='id')"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        assert (
+            await ds.allowed(
+                action="view-table",
+                resource=TableResource("data", "secret_fts"),
+            )
+            is expected
+        )
+        response = await ds.client.get("/data/secret_fts.json?_shape=array")
+        assert response.status_code == (200 if expected else 403)
+        if not expected:
+            assert "ISSUE_17_MATRIX_SECRET" not in response.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_derived_tables_propagate_private_flag_and_route_permissions():
+    actor = {"id": "reader"}
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+                "view-table": True,
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": {"id": "reader"}}}
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database("issue_17_private_flag", name="data")
+    await db.execute_write("create table secret(id integer primary key, body text)")
+    await db.execute_write("insert into secret(body) values ('PRIVATE')")
+    await db.execute_write(
+        "create virtual table secret_fts using fts5("
+        "body, content='secret', content_rowid='id')"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        actor_page = await ds.allowed_resources(
+            "view-table", actor, parent="data", include_is_private=True, limit=1000
+        )
+        actor_resources = {
+            resource.child: resource for resource in actor_page.resources
+        }
+        derived_names = set(await db.derived_table_dependencies())
+        assert "secret_fts" in actor_resources
+        assert actor_resources["secret_fts"].private
+        # Shadow tables depend on the already-derived external-content FTS
+        # table, so they remain unavailable even to the permitted reader.
+        assert not (derived_names - {"secret_fts"}).intersection(actor_resources)
+
+        anonymous_page = await ds.allowed_resources(
+            "view-table", parent="data", limit=1000
+        )
+        anonymous_names = {resource.child for resource in anonymous_page.resources}
+        assert not derived_names.intersection(anonymous_names)
+
+        for path in (
+            "/data/secret_fts.json?_facet=body",
+            "/data/secret_fts.csv",
+            "/data/secret_fts/-/autocomplete?q=PRIVATE",
+            "/data/secret_fts/-/schema.json",
+        ):
+            denied = await ds.client.get(path)
+            assert denied.status_code == 403
+            allowed = await ds.client.get(path, actor=actor)
+            assert allowed.status_code == 200
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_cyclic_derived_table_dependencies_fail_closed():
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database("issue_17_cycle", name="data")
+    await db.execute_write(
+        "create virtual table first_fts using fts5(body, content='second_fts')"
+    )
+    await db.execute_write(
+        "create virtual table second_fts using fts5(body, content='first_fts')"
+    )
+    await ds.invoke_startup()
+
+    try:
+        for table in ("first_fts", "second_fts"):
+            assert (
+                await ds.allowed(
+                    action="view-table", resource=TableResource("data", table)
+                )
+                is False
+            )
+
+        page = await ds.allowed_resources("view-table", parent="data", limit=1000)
+        allowed_names = {resource.child for resource in page.resources}
+        assert "first_fts" not in allowed_names
+        assert "second_fts" not in allowed_names
+    finally:
+        ds.close()

tests/test_html.py

@@ -1,16 +1,19 @@
-from bs4 import BeautifulSoup as Soup
-from datasette.app import Datasette
-from datasette.utils import allowed_pragmas
-from .fixtures import make_app_client
-from .utils import assert_footer_links, inner_html
 import copy
 import hashlib
 import json
 import pathlib
-import pytest
 import re
 import urllib.parse
 
+import pytest
+from bs4 import BeautifulSoup as Soup
+
+from datasette.app import Datasette
+from datasette.utils import allowed_pragmas
+
+from .fixtures import make_app_client
+from .utils import assert_footer_links, inner_html
+
 
 def test_homepage(app_client_two_attached_databases):
     response = app_client_two_attached_databases.get("/")
@@ -33,8 +36,10 @@ def test_homepage(app_client_two_attached_databases):
     h2 = soup.select("h2")[0]
     assert "extra database" == h2.text.strip()
     counts_p, links_p = h2.find_all_next("p")[:2]
+    # Shadow tables of the external-content index are denied, so they do not
+    # contribute to the table or row totals.
     assert (
-        "2 rows in 1 table, 5 rows in 4 hidden tables, 1 view" == counts_p.text.strip()
+        "2 rows in 1 table, 2 rows in 1 hidden table, 1 view" == counts_p.text.strip()
     )
     # We should only show visible, not hidden tables here:
     table_links = [
@@ -46,6 +51,36 @@ def test_homepage(app_client_two_attached_databases):
     ] == table_links
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "sql,expected",
+    (
+        (["create view one as select 1 as n"], "0 tables, 1 view"),
+        (
+            ["create view one as select 1 as n", "create view two as select 2 as n"],
+            "0 tables, 2 views",
+        ),
+        (
+            ["create table t (id integer primary key)", "create view v as select 1"],
+            "0 rows in 1 table, 1 view",
+        ),
+    ),
+)
+async def test_homepage_database_summary_separators(sql, expected):
+    # https://github.com/simonw/datasette/issues/2012
+    ds = Datasette()
+    await ds.invoke_startup()
+    db = ds.add_memory_database("summary_separators")
+    for statement in sql:
+        await db.execute_write(statement)
+    response = await ds.client.get("/")
+    assert response.status_code == 200
+    soup = Soup(response.text, "html.parser")
+    h2 = next(h2 for h2 in soup.select("h2") if h2.text.strip() == "summary_separators")
+    counts_p = h2.find_next("p")
+    assert " ".join(counts_p.text.split()) == expected
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("path", ("/", "/-/"))
 async def test_homepage_alternative_location(path, tmp_path_factory):
@@ -142,9 +177,7 @@ def test_static_mounts_hash_cache_control():
         )
 
         incorrect_hash = hashlib.sha256(b"incorrect").hexdigest()[:12]
-        response = client.get(
-            "/custom-static/test_html.py?_hash={}".format(incorrect_hash)
-        )
+        response = client.get(f"/custom-static/test_html.py?_hash={incorrect_hash}")
         assert response.status_code == 200
         assert "cache-control" not in response.headers
 
@@ -219,11 +252,9 @@ async def test_disallowed_custom_sql_pragma(ds_client):
         "/fixtures/-/query?sql=SELECT+*+FROM+pragma_not_on_allow_list('idx52')"
     )
     assert response.status_code == 400
-    pragmas = ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas)
+    pragmas = ", ".join(f"pragma_{pragma}()" for pragma in allowed_pragmas)
     assert (
-        "Statement contained a disallowed PRAGMA. Allowed pragma functions are {}".format(
-            pragmas
-        )
+        f"Statement contained a disallowed PRAGMA. Allowed pragma functions are {pragmas}"
         in response.text
     )
 
@@ -569,6 +600,52 @@ async def test_database_metadata_with_custom_sql(ds_client):
     # assert_footer_links(soup)TODO(alex) ensure
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "path,metadata,expected_text,expected_href",
+    (
+        ("/", {"about": "Instance about"}, "About: Instance about", None),
+        (
+            "/",
+            {"about_url": "https://example.com/"},
+            "About: https://example.com/",
+            "https://example.com/",
+        ),
+        (
+            "/data",
+            {"databases": {"data": {"about": "Database about"}}},
+            "About: Database about",
+            None,
+        ),
+        (
+            "/data/t",
+            {"databases": {"data": {"tables": {"t": {"about": "Table about"}}}}},
+            "About: Table about",
+            None,
+        ),
+    ),
+)
+async def test_about_without_source_or_license(
+    path, metadata, expected_text, expected_href
+):
+    # https://github.com/simonw/datasette/issues/512
+    ds = Datasette(metadata=metadata)
+    db = ds.add_memory_database("test_about_without_source_or_license", name="data")
+    await db.execute_write("create table if not exists t (id integer primary key)")
+    response = await ds.client.get(path)
+    assert response.status_code == 200
+    soup = Soup(response.text, "html.parser")
+    about_p = soup.select_one("section.content p:-soup-contains('About:')")
+    assert about_p is not None
+    # No leading separator, since there is no license or source before it
+    assert " ".join(about_p.text.split()) == expected_text
+    links = about_p.find_all("a")
+    if expected_href:
+        assert [a["href"] for a in links] == [expected_href]
+    else:
+        assert links == []
+
+
 def test_database_download_for_immutable():
     with make_app_client(is_immutable=True) as client:
         assert not client.ds.databases["fixtures"].is_mutable
@@ -778,8 +855,8 @@ def test_stored_query_show_hide_metadata_option(
         },
         memory=True,
     ) as client:
-        expected_show_hide_fragment = '(<a href="{}">{}</a>)'.format(
-            expected_show_hide_link, expected_show_hide_text
+        expected_show_hide_fragment = (
+            f'(<a href="{expected_show_hide_link}">{expected_show_hide_text}</a>)'
         )
         response = client.get("/_memory/one" + querystring)
         html = response.text
@@ -788,10 +865,7 @@ def test_stored_query_show_hide_metadata_option(
         )[0]
         assert show_hide_fragment == expected_show_hide_fragment
         if expected_hidden:
-            assert (
-                '<input type="hidden" name="{}" value="1">'.format(expected_hidden)
-                in html
-            )
+            assert f'<input type="hidden" name="{expected_hidden}" value="1">' in html
         else:
             assert '<input type="hidden" ' not in html
 
@@ -916,6 +990,7 @@ def test_debug_context_includes_extra_template_vars():
         "/fixtures/-/query?sql=select+1",
         "/-/api",
         "/-/patterns",
+        "/-/patterns/menus",
     ],
 )
 @pytest.mark.parametrize("use_prefix", (True, False))
@@ -1194,13 +1269,9 @@ async def test_alternate_url_json(ds_client, path, expected):
     response = await ds_client.get(path)
     assert response.status_code == 200
     link = response.headers["link"]
-    assert link == '<{}>; rel="alternate"; type="application/json+datasette"'.format(
-        expected
-    )
+    assert link == f'<{expected}>; rel="alternate"; type="application/json+datasette"'
     assert (
-        '<link rel="alternate" type="application/json+datasette" href="{}">'.format(
-            expected
-        )
+        f'<link rel="alternate" type="application/json+datasette" href="{expected}">'
         in response.text
     )
 
@@ -1208,7 +1279,13 @@ async def test_alternate_url_json(ds_client, path, expected):
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "path",
-    ("/-/patterns", "/-/messages", "/-/allow-debug", "/fixtures.db"),
+    (
+        "/-/patterns",
+        "/-/patterns/menus",
+        "/-/messages",
+        "/-/allow-debug",
+        "/fixtures.db",
+    ),
 )
 async def test_no_alternate_url_json(ds_client, path):
     response = await ds_client.get(path)
@@ -1292,8 +1369,8 @@ async def test_database_color(ds_client):
     expected_color = ds_client.ds.get_database("fixtures").color
     # Should be something like #9403e5
     expected_fragments = (
-        "10px solid #{}".format(expected_color),
-        "border-color: #{}".format(expected_color),
+        f"10px solid #{expected_color}",
+        f"border-color: #{expected_color}",
     )
     assert len(expected_color) == 6
     for path in (

tests/test_http_span.py

@@ -0,0 +1,739 @@
+"""
+Tests for the HTTP request span created by TelemetryMiddleware and the
+`http.route` enrichment added by the router.
+"""
+
+import asyncio
+import itertools
+import json
+import subprocess
+import sys
+import textwrap
+import time
+
+import pytest
+import pytest_asyncio
+
+pytest.importorskip("opentelemetry.sdk")
+
+from opentelemetry.trace import (
+    NonRecordingSpan,
+    SpanContext,
+    SpanKind,
+    StatusCode,
+    TraceFlags,
+)
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.telemetry import (
+    REQUEST_SPAN_SCOPE_KEY,
+    TelemetryMiddleware,
+    request_span,
+    tracer,
+)
+from datasette.utils import resolve_routes
+
+# Named in-memory databases are shared between instances, so each fixture
+# needs a unique name.
+_names = itertools.count()
+
+
+PLUGIN_MIDDLEWARE_SPAN = "test.plugin.middleware"
+
+
+class _MiddlewarePlugin:
+    "A plugin asgi_wrapper() that creates a span."
+
+    __name__ = "HttpSpanMiddlewarePlugin"
+
+    @hookimpl
+    def asgi_wrapper(self, datasette):
+        def wrap(app):
+            async def wrapped(scope, receive, send):
+                with tracer.start_as_current_span(PLUGIN_MIDDLEWARE_SPAN):
+                    await app(scope, receive, send)
+
+            return wrapped
+
+        return wrap
+
+
+class _RaisingMiddlewarePlugin:
+    """
+    A plugin asgi_wrapper() that raises. `route_path` turns most exceptions
+    into a 500, so this is how an exception reaches the request span.
+    """
+
+    __name__ = "HttpSpanRaisingMiddlewarePlugin"
+
+    def __init__(self, call_app_first):
+        self.call_app_first = call_app_first
+
+    @hookimpl
+    def asgi_wrapper(self, datasette):
+        call_app_first = self.call_app_first
+
+        def wrap(app):
+            async def wrapped(scope, receive, send):
+                if call_app_first:
+                    await app(scope, receive, send)
+                raise RuntimeError("wrapper exploded")
+
+            return wrapped
+
+        return wrap
+
+
+class _BoomPlugin:
+    "A route that raises, which route_path turns into a 500."
+
+    __name__ = "HttpSpanBoomPlugin"
+
+    @hookimpl
+    def register_routes(self):
+        return [(r"^/-/http-span-boom$", lambda: 1 / 0)]
+
+
+@pytest_asyncio.fixture
+async def ds():
+    name = f"httpspan{next(_names)}"
+    instance = Datasette(memory=True)
+    instance.add_memory_database(name)
+    await instance.invoke_startup()
+    await instance.get_database(name).execute_write(
+        "create table t (id integer primary key, v text)"
+    )
+    instance.db_name = name
+    try:
+        yield instance
+    finally:
+        instance.close()
+
+
+@pytest_asyncio.fixture
+async def ds_paging():
+    """
+    An instance whose table is bigger than `max_returned_rows`, so a
+    `?_stream=1` export runs queries for later pages during the body send.
+    """
+    name = f"httpspanpaging{next(_names)}"
+    # Both settings are needed: lowering only max_returned_rows gives a
+    # single page with no `next` token.
+    instance = Datasette(
+        memory=True, settings={"max_returned_rows": 5, "default_page_size": 3}
+    )
+    instance.add_memory_database(name)
+    await instance.invoke_startup()
+    db = instance.get_database(name)
+    await db.execute_write("create table t (id integer primary key, v text)")
+    await db.execute_write_many(
+        "insert into t (id, v) values (?, ?)", [[i, f"v{i}"] for i in range(40)]
+    )
+    instance.db_name = name
+    try:
+        yield instance
+    finally:
+        instance.close()
+
+
+def _server_spans(otel_spans):
+    return [
+        span for span in otel_spans.get_finished_spans() if span.kind is SpanKind.SERVER
+    ]
+
+
+def _route_for(ds, path):
+    "The compiled pattern Datasette's own router resolves `path` to."
+    match, _view = resolve_routes(ds._routes(), path)
+    assert match is not None, f"{path} matches no route"
+    return match.re.pattern
+
+
+@pytest.mark.asyncio
+async def test_plugin_asgi_wrapper_middleware_runs_inside_the_request_span(
+    ds, otel_spans
+):
+    """
+    Spans created by plugin asgi_wrapper() middleware are children of the
+    request span.
+    """
+    ds.pm.register(_MiddlewarePlugin(), name="httpspan-middleware")
+    try:
+        otel_spans.clear()
+        response = await ds.client.get(f"/{ds.db_name}/t")
+        assert response.status_code == 200
+    finally:
+        ds.pm.unregister(name="httpspan-middleware")
+
+    spans = otel_spans.get_finished_spans()
+    server = [span for span in spans if span.kind is SpanKind.SERVER]
+    assert len(server) == 1, "expected exactly one SERVER span per request"
+    server_span = server[0]
+    assert server_span.parent is None, "the request span should be the trace root"
+
+    plugin_spans = [span for span in spans if span.name == PLUGIN_MIDDLEWARE_SPAN]
+    assert len(plugin_spans) == 1
+    assert plugin_spans[0].parent is not None
+    assert plugin_spans[0].parent.span_id == server_span.context.span_id
+    assert plugin_spans[0].context.trace_id == server_span.context.trace_id
+
+    # Database spans are in the same trace.
+    queries = [span for span in spans if span.name == "db.query"]
+    assert queries, "a table page should have issued at least one query"
+    for query in queries:
+        assert query.context.trace_id == server_span.context.trace_id
+
+
+@pytest.mark.asyncio
+async def test_unrecognised_method_is_clamped(ds, otel_spans):
+    """
+    Unknown methods are recorded as `_OTHER` in both the attribute and the
+    span name, which the router rebuilds from the raw `request.method`.
+    """
+    otel_spans.clear()
+    await ds.client.request("FROB", f"/{ds.db_name}/t")
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].attributes["http.request.method"] == "_OTHER"
+    assert server[0].name == f"_OTHER {server[0].attributes['http.route']}"
+
+
+@pytest.mark.asyncio
+async def test_known_method_is_not_clamped(ds, otel_spans):
+    "Known methods are recorded unchanged."
+    otel_spans.clear()
+    await ds.client.get(f"/{ds.db_name}/t")
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].attributes["http.request.method"] == "GET"
+    assert server[0].name == f"GET {server[0].attributes['http.route']}"
+
+
+@pytest.mark.asyncio
+async def test_the_query_string_is_never_recorded(ds, otel_spans):
+    "No attribute on any span contains the query string."
+    marker = "canary-9f2b1c"
+    otel_spans.clear()
+    await ds.client.get(f"/{ds.db_name}/t?_facet=v&_nosuch={marker}")
+    spans = otel_spans.get_finished_spans()
+    assert _server_spans(otel_spans), "no request span was emitted"
+    leaked = [
+        f"{span.name} -> {key}={value!r}"
+        for span in spans
+        for key, value in (span.attributes or {}).items()
+        if marker in str(value) or key == "url.query"
+    ]
+    assert not leaked, "the query string reached a span attribute: " + ", ".join(leaked)
+
+
+@pytest.mark.asyncio
+async def test_url_path_is_recorded_without_the_query_string(ds, otel_spans):
+    otel_spans.clear()
+    await ds.client.get(f"/{ds.db_name}/t?_facet=v")
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].attributes["url.path"] == f"/{ds.db_name}/t"
+
+
+@pytest.mark.asyncio
+async def test_escaping_exception_sets_error_type_and_reraises(ds, otel_spans):
+    """
+    An exception that escapes `route_path` is recorded and re-raised. No
+    response started, so no status code is recorded.
+    """
+    ds.pm.register(
+        _RaisingMiddlewarePlugin(call_app_first=False), name="httpspan-raiser"
+    )
+    try:
+        otel_spans.clear()
+        with pytest.raises(RuntimeError):
+            await ds.client.get(f"/{ds.db_name}/t")
+    finally:
+        ds.pm.unregister(name="httpspan-raiser")
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].attributes["error.type"] == "RuntimeError"
+    assert "http.response.status_code" not in server[0].attributes
+    assert server[0].status.status_code is StatusCode.ERROR
+
+
+@pytest.mark.asyncio
+async def test_an_escaping_exception_beats_the_status_code_for_error_type(
+    ds, otel_spans
+):
+    """
+    A 500 response followed by an exception records the exception class as
+    `error.type`, not "500".
+    """
+    ds.pm.register(_BoomPlugin(), name="httpspan-boom")
+    ds.pm.register(
+        _RaisingMiddlewarePlugin(call_app_first=True), name="httpspan-raiser"
+    )
+    try:
+        otel_spans.clear()
+        with pytest.raises(RuntimeError):
+            await ds.client.get("/-/http-span-boom")
+    finally:
+        ds.pm.unregister(name="httpspan-raiser")
+        ds.pm.unregister(name="httpspan-boom")
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].attributes["http.response.status_code"] == 500
+    assert server[0].attributes["error.type"] == "RuntimeError"
+
+
+@pytest.mark.asyncio
+async def test_a_404_is_not_an_error(ds, otel_spans):
+    """
+    A 4xx records the status code but no `error.type` or error status.
+    `/no-such-database-at-all` matches the database route, so `http.route`
+    is still set.
+    """
+    otel_spans.clear()
+    response = await ds.client.get("/no-such-database-at-all")
+    assert response.status_code == 404
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].attributes["http.response.status_code"] == 404
+    assert "error.type" not in server[0].attributes
+    assert server[0].status.status_code is StatusCode.UNSET
+    assert "http.route" in server[0].attributes
+    assert server[0].name != "GET"
+
+
+@pytest.mark.asyncio
+async def test_an_unrouted_404_has_no_route_and_a_bare_method_name(ds, otel_spans):
+    """
+    With no matching route the span keeps the bare method name. Most missing
+    paths still match a route, so this uses a path deeper than any route.
+    """
+    otel_spans.clear()
+    response = await ds.client.get("/a/b/c/d/e")
+    assert response.status_code == 404
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].name == "GET"
+    assert "http.route" not in server[0].attributes
+    assert server[0].attributes["http.response.status_code"] == 404
+    assert server[0].status.status_code is StatusCode.UNSET
+
+
+@pytest.mark.asyncio
+async def test_only_the_first_http_response_start_is_recorded(otel_spans):
+    "The `send` wrapper records the status from the first `http.response.start`."
+
+    async def two_starts(scope, receive, send):
+        await send({"type": "http.response.start", "status": 200, "headers": []})
+        await send({"type": "http.response.start", "status": 503, "headers": []})
+        await send({"type": "http.response.body", "body": b""})
+
+    middleware = TelemetryMiddleware(two_starts)
+    scope = {
+        "type": "http",
+        "method": "GET",
+        "path": "/twice",
+        "raw_path": b"/twice",
+        "scheme": "http",
+        "headers": [],
+    }
+    otel_spans.clear()
+    await middleware(scope, None, lambda message: asyncio.sleep(0))
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].attributes["http.response.status_code"] == 200
+    assert "error.type" not in server[0].attributes
+
+
+@pytest.mark.asyncio
+async def test_lifespan_scope_passes_through_unspanned(otel_spans):
+    """
+    Lifespan scopes reach `AsgiLifespan`, which sits inside this middleware,
+    without creating a SERVER span.
+    """
+    instance = Datasette(memory=True)
+    app = instance.app()
+    events = iter([{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}])
+    sent = []
+
+    async def receive():
+        return next(events)
+
+    async def send(message):
+        sent.append(message["type"])
+
+    otel_spans.clear()
+    await app({"type": "lifespan"}, receive, send)
+    assert sent == ["lifespan.startup.complete", "lifespan.shutdown.complete"]
+    assert not _server_spans(otel_spans)
+
+
+@pytest.mark.asyncio
+async def test_http_route_is_the_compiled_pattern(ds, otel_spans):
+    "`http.route` is the compiled regex of the route Datasette's router resolves."
+    path = f"/{ds.db_name}/t"
+    expected = _route_for(ds, path)
+    otel_spans.clear()
+    assert (await ds.client.get(path)).status_code == 200
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].attributes["http.route"] == expected
+    assert server[0].name == f"GET {expected}"
+    # The raw pattern, not a prettified template:
+    assert "(?P<database>" in expected
+
+
+@pytest.mark.asyncio
+async def test_the_route_lands_on_the_request_span_not_a_plugins_current_span(
+    ds, otel_spans
+):
+    """
+    The route is set on the span the middleware started, found through the
+    ASGI scope, not on a plugin `asgi_wrapper()` span that is current during
+    routing.
+    """
+    ds.pm.register(_MiddlewarePlugin(), name="httpspan-middleware")
+    try:
+        otel_spans.clear()
+        path = f"/{ds.db_name}/t"
+        expected = _route_for(ds, path)
+        assert (await ds.client.get(path)).status_code == 200
+    finally:
+        ds.pm.unregister(name="httpspan-middleware")
+
+    spans = otel_spans.get_finished_spans()
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].attributes["http.route"] == expected
+    assert server[0].name == f"GET {expected}"
+    # The plugin's span keeps its name and has no route attribute.
+    plugin_spans = [span for span in spans if span.name == PLUGIN_MIDDLEWARE_SPAN]
+    assert len(plugin_spans) == 1
+    assert "http.route" not in (plugin_spans[0].attributes or {})
+
+
+@pytest.mark.asyncio
+async def test_request_span_attributes(ds, otel_spans):
+    "The attributes recorded for an ordinary request."
+    path = f"/{ds.db_name}/t"
+    otel_spans.clear()
+    assert (await ds.client.get(path)).status_code == 200
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    attributes = server[0].attributes
+    assert attributes["http.request.method"] == "GET"
+    assert attributes["url.path"] == path
+    assert attributes["url.scheme"] == "http"
+    assert attributes["http.response.status_code"] == 200
+    assert attributes["http.route"] == _route_for(ds, path)
+    assert server[0].status.status_code is StatusCode.UNSET
+    # The client IP address and query string are not recorded.
+    assert "client.address" not in attributes
+    assert "url.query" not in attributes
+
+
+@pytest.mark.asyncio
+async def test_db_query_spans_are_children_of_the_request_span(ds, otel_spans):
+    """
+    Every `db.query` span descends from the request span, which is the only
+    root span.
+    """
+    otel_spans.clear()
+    assert (await ds.client.get(f"/{ds.db_name}/t?_facet=v")).status_code == 200
+    spans = otel_spans.get_finished_spans()
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    server_span = server[0]
+    assert server_span.parent is None
+
+    by_span_id = {span.context.span_id: span for span in spans}
+    roots = [span for span in spans if span.parent is None]
+    assert [span.name for span in roots] == [server_span.name], (
+        "every span from a request should hang off the request span, but these "
+        f"are roots: {sorted(span.name for span in roots)}"
+    )
+
+    queries = [span for span in spans if span.name == "db.query"]
+    assert queries, "a faceted table page should have issued queries"
+    for query in queries:
+        assert query.context.trace_id == server_span.context.trace_id
+        # Walk up to the root, which should be the request span.
+        current = query
+        seen = 0
+        while current.parent is not None:
+            current = by_span_id[current.parent.span_id]
+            seen += 1
+            assert seen < 20, "parent chain did not terminate"
+        assert current is server_span
+
+
+@pytest.mark.asyncio
+async def test_500_sets_error_status_and_error_type(ds, otel_spans):
+    """
+    `route_path` turns the exception into a 500 response, so `error.type` is
+    the status code as a string.
+    """
+    ds.pm.register(_BoomPlugin(), name="httpspan-boom")
+    try:
+        otel_spans.clear()
+        response = await ds.client.get("/-/http-span-boom")
+        assert response.status_code == 500
+    finally:
+        ds.pm.unregister(name="httpspan-boom")
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    assert server[0].attributes["http.response.status_code"] == 500
+    assert server[0].attributes["error.type"] == "500"
+    assert server[0].status.status_code is StatusCode.ERROR
+
+
+@pytest.mark.asyncio
+async def test_csv_stream_span_covers_the_body_send(ds_paging, otel_spans):
+    """
+    The request span covers a streamed CSV body, including queries for later
+    pages that run after the response has started.
+
+    Driven as raw ASGI to timestamp `http.response.start` with `time.time_ns()`,
+    the clock the SDK uses for spans.
+    """
+    app = ds_paging.app()
+    body = []
+    response_started_at = None
+
+    async def receive():
+        return {"type": "http.request", "body": b"", "more_body": False}
+
+    async def send(message):
+        nonlocal response_started_at
+        if message["type"] == "http.response.start":
+            assert message["status"] == 200
+            response_started_at = time.time_ns()
+        else:
+            body.append(message.get("body") or b"")
+
+    otel_spans.clear()
+    await app(
+        {
+            "type": "http",
+            "http_version": "1.1",
+            "method": "GET",
+            "path": f"/{ds_paging.db_name}/t.csv",
+            "raw_path": f"/{ds_paging.db_name}/t.csv".encode("latin-1"),
+            "query_string": b"_stream=1",
+            "scheme": "http",
+            "headers": [(b"host", b"localhost")],
+        },
+        receive,
+        send,
+    )
+    # 40 rows plus a header, so the export read past the first page
+    assert len(b"".join(body).decode("utf-8").strip().splitlines()) == 41
+    assert response_started_at is not None
+
+    spans = otel_spans.get_finished_spans()
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    server_span = server[0]
+    queries = [span for span in spans if span.name == "db.query"]
+    assert len(queries) > 1
+    during_body = [span for span in queries if span.start_time > response_started_at]
+    assert during_body, (
+        "no query ran after the response started, so this workload cannot "
+        "distinguish a span that covers the body send from one that ends when "
+        "the handler returns - the export is not paging"
+    )
+    last_query_end = max(span.end_time for span in queries)
+    assert server_span.end_time > last_query_end, (
+        "the request span ended before the last query of a streaming export - "
+        "it is not covering the response body"
+    )
+    for query in queries:
+        assert query.context.trace_id == server_span.context.trace_id
+
+
+@pytest.mark.asyncio
+async def test_inbound_traceparent_becomes_the_parent(ds, otel_spans):
+    """
+    An inbound `traceparent` header continues the caller's trace. It uses the
+    sampled flag (`-01`) because the SDK's default sampler is parent-based.
+    """
+    trace_id = "4bf92f3577b34da6a3ce929d0e0e4736"
+    parent_span_id = "00f067aa0ba902b7"
+    otel_spans.clear()
+    response = await ds.client.get(
+        f"/{ds.db_name}/t",
+        headers={"traceparent": f"00-{trace_id}-{parent_span_id}-01"},
+    )
+    assert response.status_code == 200
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    server_span = server[0]
+    assert f"{server_span.context.trace_id:032x}" == trace_id
+    assert server_span.parent is not None
+    assert f"{server_span.parent.span_id:016x}" == parent_span_id
+    assert server_span.parent.is_remote
+    # Database spans are in the caller's trace too.
+    queries = [
+        span for span in otel_spans.get_finished_spans() if span.name == "db.query"
+    ]
+    assert queries
+    for query in queries:
+        assert f"{query.context.trace_id:032x}" == trace_id
+
+
+@pytest.mark.asyncio
+async def test_user_supplied_sql_in_the_query_string_is_never_recorded(ds, otel_spans):
+    """
+    SQL from `?sql=` is not recorded on the request span or in any `url.*`
+    or `http.*` attribute. `db.query.text` is expected to contain it.
+    """
+    marker = "secret_marker_5b1f"
+    otel_spans.clear()
+    # `/{db}?sql=` redirects to the query view, so request that directly.
+    response = await ds.client.get(f"/{ds.db_name}/-/query?sql=select+'{marker}'")
+    assert response.status_code == 200
+    spans = otel_spans.get_finished_spans()
+    server = _server_spans(otel_spans)
+    assert len(server) == 1
+    leaked = [
+        f"{span.name} -> {key}={value!r}"
+        for span in spans
+        for key, value in (span.attributes or {}).items()
+        if (span is server[0] or str(key).startswith(("url.", "http.")))
+        and (marker in str(value) or str(key) == "url.query")
+    ]
+    assert not leaked, "the query string reached a span attribute: " + ", ".join(leaked)
+    # Confirm the query ran with the marker.
+    assert marker in response.text
+
+
+def test_request_span_skips_a_valid_but_non_recording_span():
+    """
+    `request_span()` returns None for a `NonRecordingSpan` with a valid remote
+    span context, which is what an inbound `traceparent` produces with no
+    provider installed.
+    """
+    remote = SpanContext(
+        trace_id=0x4BF92F3577B34DA6A3CE929D0E0E4736,
+        span_id=0x00F067AA0BA902B7,
+        is_remote=True,
+        trace_flags=TraceFlags(TraceFlags.SAMPLED),
+    )
+    assert remote.is_valid
+    non_recording = NonRecordingSpan(remote)
+    assert non_recording.is_recording() is False
+    assert request_span({REQUEST_SPAN_SCOPE_KEY: non_recording}) is None
+    # No span in the scope and no current span:
+    assert request_span({}) is None
+    # A recording span is returned:
+    with tracer.start_as_current_span("test.request_span.recording") as span:
+        assert request_span({REQUEST_SPAN_SCOPE_KEY: span}) is span
+        # Falls back to the current span, such as one created by another
+        # SERVER instrumentation:
+        assert request_span({}) is span
+
+
+NO_PROVIDER_PROGRAM = textwrap.dedent("""
+    import asyncio, json, sys
+
+    from datasette.telemetry import TelemetryMiddleware
+
+    seen = {}
+
+
+    async def inner(scope, receive, send):
+        seen.setdefault("sends", []).append(send)
+        seen.setdefault("scopes", []).append(scope)
+        await send({"type": "http.response.start", "status": 200, "headers": []})
+        await send({"type": "http.response.body", "body": b""})
+
+
+    async def real_send(message):
+        pass
+
+
+    async def main():
+        middleware = TelemetryMiddleware(inner)
+        for headers in ([], [(b"traceparent", b"00-" + b"a" * 32 + b"-" + b"b" * 16 + b"-01")]):
+            await middleware(
+                {
+                    "type": "http",
+                    "method": "GET",
+                    "path": "/",
+                    "raw_path": b"/",
+                    "scheme": "http",
+                    "headers": headers,
+                },
+                None,
+                real_send,
+            )
+        print(
+            json.dumps(
+                {
+                    "unwrapped": [send is real_send for send in seen["sends"]],
+                    "scope_keys": [
+                        "datasette.telemetry.request_span" in scope
+                        for scope in seen["scopes"]
+                    ],
+                    "sdk_imported": any(
+                        name.startswith("opentelemetry.sdk") for name in sys.modules
+                    ),
+                }
+            )
+        )
+
+
+    asyncio.run(main())
+    """)
+
+
+def test_no_provider_takes_the_fast_path():
+    """
+    With no `TracerProvider` installed the middleware passes the original
+    `send` to the application, including for requests with a `traceparent`.
+
+    Runs in a subprocess because the suite installs a provider for the whole
+    process. conftest.py moves this test to the front of the run by name.
+    """
+    result = subprocess.run(
+        [sys.executable, "-c", NO_PROVIDER_PROGRAM],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    report = json.loads(result.stdout)
+    assert report["sdk_imported"] is False, "the SDK loaded in a fresh interpreter"
+    assert report["unwrapped"] == [True, True], (
+        "the middleware wrapped `send` with no provider installed; the second "
+        "entry is the inbound-traceparent case, which fails if the fast path "
+        "is guarded on is_valid instead of is_recording()"
+    )
+    # Nothing is stored in the scope either.
+    assert report["scope_keys"] == [False, False]
+
+
+@pytest.mark.asyncio
+async def test_internal_client_requests_are_marked(ds, otel_spans):
+    """
+    `datasette.internal_client` is set on SERVER spans for `datasette.client`
+    requests, but not for requests made directly to the ASGI app.
+    """
+    otel_spans.clear()
+    assert (await ds.client.get("/")).status_code == 200
+    server = _server_spans(otel_spans)
+    assert server
+    assert all(
+        span.attributes.get("datasette.internal_client") is True for span in server
+    )
+
+    import httpx2
+
+    transport = httpx2.ASGITransport(app=ds.app())
+    async with httpx2.AsyncClient(
+        transport=transport, base_url="http://localhost"
+    ) as client:
+        otel_spans.clear()
+        assert (await client.get("/")).status_code == 200
+    server = _server_spans(otel_spans)
+    assert server
+    assert all("datasette.internal_client" not in span.attributes for span in server)

tests/test_internal_db.py

@@ -1,6 +1,7 @@
-import pytest
 import sqlite3
 
+import pytest
+
 from datasette.utils import escape_sqlite
 from datasette.utils.internal_db import INTERNAL_DB_SCHEMA_SQL
 
@@ -137,7 +138,7 @@ async def test_internal_foreign_key_references(ds_client):
             return {
                 row[1]
                 for row in conn.execute(
-                    "PRAGMA table_info({})".format(escape_sqlite(table_name))
+                    f"PRAGMA table_info({escape_sqlite(table_name)})"
                 ).fetchall()
             }
 
@@ -147,7 +148,7 @@ async def test_internal_foreign_key_references(ds_client):
                 for _, name in sorted(
                     (row[5], row[1])
                     for row in conn.execute(
-                        "PRAGMA table_info({})".format(escape_sqlite(table_name))
+                        f"PRAGMA table_info({escape_sqlite(table_name)})"
                     ).fetchall()
                     if row[5]
                 )
@@ -159,7 +160,7 @@ async def test_internal_foreign_key_references(ds_client):
 
         for table_name in table_names:
             foreign_key_rows = conn.execute(
-                "PRAGMA foreign_key_list({})".format(escape_sqlite(table_name))
+                f"PRAGMA foreign_key_list({escape_sqlite(table_name)})"
             ).fetchall()
             foreign_keys_by_id = {}
             for foreign_key in foreign_key_rows:
@@ -169,25 +170,16 @@ async def test_internal_foreign_key_references(ds_client):
                 foreign_key_rows.sort(key=lambda row: row[1])
                 other_table = foreign_key_rows[0][2]
                 other_columns = [row[4] for row in foreign_key_rows]
-                message = 'Column "{}.{}" references other table "{}" which does not exist'.format(
-                    table_name, foreign_key_rows[0][3], other_table
-                )
+                message = f'Column "{table_name}.{foreign_key_rows[0][3]}" references other table "{other_table}" which does not exist'
                 assert other_table in table_names, message + " (bad table)"
                 if all(other_column is None for other_column in other_columns):
                     other_columns = primary_keys_for_table(other_table)
-                length_message = 'Foreign key from "{}" to "{}" has {} columns but references {} columns'.format(
-                    table_name,
-                    other_table,
-                    len(foreign_key_rows),
-                    len(other_columns),
-                )
+                length_message = f'Foreign key from "{table_name}" to "{other_table}" has {len(foreign_key_rows)} columns but references {len(other_columns)} columns'
                 assert len(other_columns) == len(foreign_key_rows), length_message
 
                 for foreign_key, other_column in zip(foreign_key_rows, other_columns):
                     column = foreign_key[3]
-                    message = 'Column "{}.{}" references other column "{}.{}" which does not exist'.format(
-                        table_name, column, other_table, other_column
-                    )
+                    message = f'Column "{table_name}.{column}" references other column "{other_table}.{other_column}" which does not exist'
                     assert other_column in columns_by_table[other_table], (
                         message + " (bad column)"
                     )
@@ -245,10 +237,10 @@ async def test_stale_catalog_entry_database_fix(tmp_path):
 
 @pytest.mark.asyncio
 async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path):
-    from datasette.app import Datasette
-
     import sqlite3
 
+    from datasette.app import Datasette
+
     internal_db_path = str(tmp_path / "internal.db")
     alpha_db_path = str(tmp_path / "alpha.db")
     bravo_db_path = str(tmp_path / "bravo.db")
@@ -293,10 +285,10 @@ async def test_stale_catalog_child_entries_removed_for_missing_database(tmp_path
 
 @pytest.mark.asyncio
 async def test_orphan_stale_catalog_child_entries_removed(tmp_path):
-    from datasette.app import Datasette
-
     import sqlite3
 
+    from datasette.app import Datasette
+
     internal_db_path = str(tmp_path / "internal.db")
     alpha_db_path = str(tmp_path / "alpha.db")
 

tests/test_internals_database.py

@@ -3,16 +3,30 @@ Tests for the datasette.database.Database class
 """
 
 import asyncio
-from types import SimpleNamespace
-from datasette.app import Datasette
-from datasette.database import Database, ExecuteWriteResult, Results, MultipleValues
-from datasette.database import DatasetteClosedError
-from datasette.database import _deliver_write_result
-from datasette.utils.sqlite import sqlite3, supports_returning
-from datasette.utils import Column
-import pytest
-import time
+import threading
 import uuid
+from types import SimpleNamespace
+
+import pytest
+import sqlite_utils
+from opentelemetry import context as otel_context_api
+
+from datasette.app import Datasette
+from datasette.database import (
+    Database,
+    DatasetteClosedError,
+    ExecuteWriteResult,
+    MultipleValues,
+    QueryInterrupted,
+    Results,
+    _deliver_write_result,
+)
+from datasette.utils import Column
+from datasette.utils.sqlite import (
+    sqlite3,
+    sqlite_derived_table_dependencies,
+    supports_returning,
+)
 
 requires_sqlite_returning = pytest.mark.skipif(
     not supports_returning(), reason="SQLite does not support RETURNING"
@@ -31,6 +45,31 @@ async def test_execute(db):
     assert 15 == len(results)
 
 
+@pytest.mark.asyncio
+async def test_derived_dependency_cache_survives_failed_refresh(monkeypatch):
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.derived_table_dependencies()
+    previous_cache = db._cached_derived_table_dependencies
+    await db.execute_write("create table dependency_cache_refresh (id integer)")
+
+    class UnavailableSchema:
+        def execute(self, sql):
+            raise sqlite3.DatabaseError("schema temporarily unavailable")
+
+    with monkeypatch.context() as patch:
+        patch.setattr(
+            "datasette.database.sqlite_derived_table_dependencies",
+            lambda conn: sqlite_derived_table_dependencies(UnavailableSchema()),
+        )
+        with pytest.raises(sqlite3.DatabaseError, match="schema temporarily"):
+            await db.derived_table_dependencies()
+    assert db._cached_derived_table_dependencies == previous_cache
+
+    await db.derived_table_dependencies()
+    assert db._cached_derived_table_dependencies[0] != previous_cache[0]
+
+
 @pytest.mark.asyncio
 async def test_results_first(db):
     assert None is (await db.execute("select * from facetable where pk > 100")).first()
@@ -43,7 +82,7 @@ async def test_results_first(db):
 @pytest.mark.parametrize("expected", (True, False))
 async def test_results_bool(db, expected):
     where = "" if expected else "where pk = 0"
-    results = await db.execute("select * from facetable {}".format(where))
+    results = await db.execute(f"select * from facetable {where}")
     assert bool(results) is expected
 
 
@@ -471,6 +510,28 @@ async def test_view_names(db):
     ]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("num_sql_threads", [0, 3])
+async def test_execute_write_zero_time_limit(num_sql_threads):
+    ds = Datasette(settings={"num_sql_threads": num_sql_threads})
+    db = ds.add_memory_database(uuid.uuid4().hex, name="write_limits")
+    try:
+        await ds.invoke_startup()
+        await db.execute_write("create table items(value integer)")
+        # Zero expires at the first SQLite progress callback, regardless of speed.
+        with pytest.raises(QueryInterrupted):
+            await db.execute_write(
+                "insert into items(value) values (1)", time_limit_ms=0
+            )
+        # No new timeout handler: the interrupted operation must have cleared it.
+        await db.execute_write(
+            "insert into items(value) values (2)", time_limit_ms=None
+        )
+        assert (await db.execute("select value from items")).single_value() == 2
+    finally:
+        ds.close()
+
+
 @pytest.mark.asyncio
 async def test_execute_write_block_true(db):
     result = await db.execute_write(
@@ -615,7 +676,7 @@ async def test_execute_write_block_false(db):
         "update roadside_attractions set name = ? where pk = ?",
         ["Mystery!", 1],
     )
-    time.sleep(0.1)
+    await asyncio.sleep(0.1)
     rows = await db.execute("select name from roadside_attractions where pk = 1")
     assert "Mystery!" == rows.rows[0][0]
 
@@ -633,7 +694,7 @@ async def test_execute_write_with_returning_block_false(db):
     )
 
     assert isinstance(task_id, uuid.UUID)
-    time.sleep(0.1)
+    await asyncio.sleep(0.1)
     assert (
         await db.execute("select name from write_returning_block_false")
     ).single_value() == "Cleo"
@@ -698,6 +759,33 @@ async def test_execute_write_fn_block_false(db):
     assert isinstance(task_id, uuid.UUID)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("disable_threads", (False, True))
+async def test_execute_write_fn_block_false_returns_uuid(tmp_path, disable_threads):
+    # block=False is documented to return "a UUID representing the queued task".
+    # With num_sql_threads=0 there is no write thread, so the non-threaded branch
+    # has to satisfy the same contract as the threaded one.
+    settings = {"num_sql_threads": 0} if disable_threads else {}
+    ds = Datasette([], memory=True, settings=settings)
+    await ds.invoke_startup()
+    db = ds.add_memory_database("test_block_false")
+    await db.execute_write(
+        "create table if not exists t (id integer primary key, v text)"
+    )
+
+    def write_fn(conn):
+        conn.execute("insert into t (v) values ('a')")
+        # Returns None, like most write functions.
+
+    task_id = await db.execute_write_fn(write_fn, block=False)
+
+    assert isinstance(task_id, uuid.UUID)
+    # Distinct per call, so a caller can tell two queued tasks apart.
+    second = await db.execute_write_fn(write_fn, block=False)
+    assert isinstance(second, uuid.UUID)
+    assert second != task_id
+
+
 @pytest.mark.asyncio
 async def test_execute_write_fn_block_true(db):
     def write_fn(conn):
@@ -718,15 +806,51 @@ async def test_execute_write_fn_exception(db):
         await db.execute_write_fn(write_fn)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("num_sql_threads", (0, 1))
+async def test_execute_write_fn_sqlite_utils_transaction(tmp_path, num_sql_threads):
+    # A write inside a failing Datasette task must never become visible or
+    # survive the rollback. Exercise both the synchronous and writer-thread
+    # paths against a file-backed database so a second connection can observe
+    # committed state independently.
+    db_path = tmp_path / "test.db"
+    sqlite3.connect(db_path).close()
+    ds = Datasette([str(db_path)], settings={"num_sql_threads": num_sql_threads})
+    db = ds.get_database("test")
+    await db.execute_write("create table items (id integer primary key)")
+    # This reader is used inside the write callback, which may run on another
+    # thread, but it is never accessed concurrently.
+    reader = sqlite3.connect(db_path, check_same_thread=False)
+
+    def insert_then_fail(conn):
+        # Datasette must open the outer transaction before sqlite-utils writes.
+        assert conn.in_transaction
+        sqlite_utils.Database(conn)["items"].insert({"id": 1})
+        # If sqlite-utils committed its own transaction, this would return 1.
+        assert reader.execute("select count(*) from items").fetchone()[0] == 0
+        # Simulate a later step failing after the sqlite-utils write succeeded.
+        raise ValueError("deliberate")
+
+    try:
+        with pytest.raises(ValueError, match="deliberate"):
+            await db.execute_write_fn(insert_then_fail)
+        # The outer transaction must roll back the sqlite-utils write as well.
+        assert reader.execute("select count(*) from items").fetchone()[0] == 0
+    finally:
+        reader.close()
+        db.close()
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize("param_name", ["conn", "connection", "db", "c"])
 async def test_execute_write_fn_accepts_any_single_param_name(db, param_name):
     # Plugins historically relied on the fact that the callback was invoked
     # positionally, so any parameter name worked. Preserve that contract.
     scope = {}
-    exec(
-        "def write_fn({0}):\n"
-        "    return {0}.execute('select 1 + 1').fetchone()[0]".format(param_name),
+    # exec() is how we build a function with a parameterized argument name
+    exec(  # noqa: S102
+        f"def write_fn({param_name}):\n"
+        f"    return {param_name}.execute('select 1 + 1').fetchone()[0]",
         scope,
     )
     write_fn = scope["write_fn"]
@@ -750,7 +874,9 @@ async def test_execute_write_fn_with_track_event(db):
 
 
 @pytest.mark.asyncio
-@pytest.mark.timeout(1)
+# func_only so the budget covers the write-thread call under test, not the
+# one-off app_client fixture setup this test may be first to trigger
+@pytest.mark.timeout(1, func_only=True)
 async def test_execute_write_fn_connection_exception(tmpdir, app_client):
     path = str(tmpdir / "immutable.db")
     conn = sqlite3.connect(path)
@@ -1178,3 +1304,103 @@ async def test_database_close_is_idempotent(tmpdir):
     # Second call should be a no-op, not raise
     db.close()
     ds._internal_database.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("num_sql_threads", [0, 2])
+@pytest.mark.parametrize("named", [False, True])
+async def test_close_releases_memory_connections(num_sql_threads, named):
+    ds = Datasette(memory=True, settings={"num_sql_threads": num_sql_threads})
+    db = ds.add_memory_database(uuid.uuid4().hex) if named else ds.get_database()
+    read_connection = await db.execute_fn(lambda conn: conn)
+    write_connection = await db.execute_write_fn(lambda conn: conn)
+    ds.close()
+    for conn in (read_connection, write_connection):
+        with pytest.raises(sqlite3.ProgrammingError, match="closed"):
+            conn.execute("select 1")
+
+
+_CONTEXT_LEAK_MARKER_KEY = "otel-context-leak-marker"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("num_sql_threads", (0, 1))
+async def test_write_thread_context_is_detached_between_tasks(
+    tmp_path, monkeypatch, num_sql_threads
+):
+    """
+    The write thread attaches each task's OpenTelemetry context and detaches
+    it before the next task, including when the task raises an exception.
+
+    Checks that each task sees the context from when it was queued, and that
+    the write thread's attach depth does not grow between tasks.
+    """
+    name = f"context_leak_test_{num_sql_threads}"
+    db_path = tmp_path / f"{name}.db"
+    sqlite3.connect(db_path).close()
+    ds = Datasette([str(db_path)], settings={"num_sql_threads": num_sql_threads})
+    db = ds.get_database(name)
+    await db.execute_write("create table t (id integer primary key)")
+
+    write_thread_name = f"_execute_writes for database {name}"
+    depth = {"value": 0}
+    real_attach = otel_context_api.attach
+    real_detach = otel_context_api.detach
+
+    def counting_attach(context):
+        token = real_attach(context)
+        if threading.current_thread().name == write_thread_name:
+            depth["value"] += 1
+        return token
+
+    def counting_detach(token):
+        real_detach(token)
+        if threading.current_thread().name == write_thread_name:
+            depth["value"] -= 1
+
+    # database.py and opentelemetry.trace both call these via the module
+    monkeypatch.setattr(otel_context_api, "attach", counting_attach)
+    monkeypatch.setattr(otel_context_api, "detach", counting_detach)
+
+    seen_markers = []
+    seen_depths = []
+
+    def probe(conn):
+        seen_markers.append(otel_context_api.get_value(_CONTEXT_LEAK_MARKER_KEY))
+        seen_depths.append(depth["value"])
+
+    def failing_probe(conn):
+        probe(conn)
+        raise ValueError("deliberate failure inside a write task")
+
+    try:
+        for i in range(5):
+            ctx = otel_context_api.set_value(_CONTEXT_LEAK_MARKER_KEY, f"marker-{i}")
+            token = real_attach(ctx)
+            try:
+                if i == 2:
+                    with pytest.raises(ValueError):
+                        await db.execute_write_fn(failing_probe)
+                else:
+                    await db.execute_write_fn(probe)
+            finally:
+                real_detach(token)
+
+        # No marker is set here, so the final probe should see None
+        assert otel_context_api.get_value(_CONTEXT_LEAK_MARKER_KEY) is None
+        await db.execute_write_fn(probe)
+    finally:
+        db.close()
+
+    assert seen_markers == [
+        "marker-0",
+        "marker-1",
+        "marker-2",
+        "marker-3",
+        "marker-4",
+        None,
+    ]
+    assert len(set(seen_depths)) == 1, (
+        f"write thread context stack grew across tasks: {seen_depths} - "
+        "a token was attached without being detached"
+    )

tests/test_internals_datasette.py

@@ -9,13 +9,15 @@ import importlib
 import os
 import sqlite3
 import time
+
+import pytest
+from itsdangerous import BadSignature
+
 from datasette import Context
-from datasette.app import Datasette, Database, ResourcesSQL
+from datasette.app import Database, Datasette, ResourcesSQL
 from datasette.database import DatasetteClosedError
 from datasette.resources import DatabaseResource
 from datasette.utils import PrefixedUrlString
-from itsdangerous import BadSignature
-import pytest
 
 
 @pytest.fixture
@@ -77,9 +79,7 @@ async def test_static_template_function_hashes_core_asset(tmp_path, monkeypatch)
     template = ds.get_jinja_environment().from_string("{{ static('demo.js') }}")
     expected_hash = hashlib.sha256(b"const demo = true;").hexdigest()[:12]
 
-    assert await template.render_async() == "/-/static/demo.js?_hash={}".format(
-        expected_hash
-    )
+    assert await template.render_async() == f"/-/static/demo.js?_hash={expected_hash}"
     assert isinstance(ds.static("demo.js"), PrefixedUrlString)
 
 
@@ -101,7 +101,7 @@ def test_static_hash_recalculated_when_cache_headers_disabled(tmp_path, monkeypa
     asset_path.write_bytes(b"let a = 2;")
 
     expected_hash = hashlib.sha256(b"let a = 2;").hexdigest()[:12]
-    assert ds.static("demo.js") == "/-/static/demo.js?_hash={}".format(expected_hash)
+    assert ds.static("demo.js") == f"/-/static/demo.js?_hash={expected_hash}"
     assert ds.static("demo.js") != first_url
 
 
@@ -114,12 +114,12 @@ def test_static_hashes_mounted_static_file(tmp_path):
     expected_hash = hashlib.sha256(b"body { color: black; }").hexdigest()[:12]
 
     assert ds.static("styles.css", mount="assets") == (
-        "/assets/styles.css?_hash={}".format(expected_hash)
+        f"/assets/styles.css?_hash={expected_hash}"
     )
 
     ds._settings["base_url"] = "/prefix/"
     assert ds.static("styles.css", mount="assets") == (
-        "/prefix/assets/styles.css?_hash={}".format(expected_hash)
+        f"/prefix/assets/styles.css?_hash={expected_hash}"
     )
 
 
@@ -144,9 +144,7 @@ def test_static_hashes_plugin_static_file(tmp_path, monkeypatch):
     expected_hash = hashlib.sha256(b"console.log('plugin');").hexdigest()[:12]
 
     assert ds.static("plugin.js", plugin="datasette_cluster_map") == (
-        "/-/static-plugins/datasette_cluster_map/plugin.js?_hash={}".format(
-            expected_hash
-        )
+        f"/-/static-plugins/datasette_cluster_map/plugin.js?_hash={expected_hash}"
     )
 
 

tests/test_internals_datasette_client.py

@@ -1,6 +1,7 @@
-import httpx
+import httpx2
 import pytest
 import pytest_asyncio
+
 from datasette.app import Datasette
 
 
@@ -42,7 +43,7 @@ async def datasette_with_permissions():
 async def test_client_methods(datasette, method, path, expected_status):
     client_method = getattr(datasette.client, method)
     response = await client_method(path)
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
     assert response.status_code == expected_status
     # Try that again using datasette.client.request
     response2 = await datasette.client.request(method, path)
@@ -62,7 +63,7 @@ async def test_client_post(datasette, prefix):
                 "message": "A message",
             },
         )
-        assert isinstance(response, httpx.Response)
+        assert isinstance(response, httpx2.Response)
         assert response.status_code == 302
         assert "ds_messages" in response.cookies
     finally:
@@ -134,7 +135,7 @@ async def test_skip_permission_checks_all_methods(datasette_with_permissions, me
     response = await client_method("/test_db.json", skip_permission_checks=True)
     # We don't check status code since some methods might not be allowed,
     # but we verify the request doesn't fail due to permissions
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
 
 
 @pytest.mark.asyncio
@@ -238,7 +239,7 @@ async def test_in_client_returns_false_outside_request(datasette):
 @pytest.mark.asyncio
 async def test_in_client_returns_true_inside_request():
     """Test that datasette.in_client() returns True inside a client request"""
-    from datasette import hookimpl, Response
+    from datasette import Response, hookimpl
 
     class TestPlugin:
         __name__ = "test_in_client_plugin"
@@ -339,7 +340,7 @@ async def test_actor_parameter_all_http_methods(datasette, method):
     client_method = getattr(datasette.client, method)
     # Just verify no TypeError about unexpected 'actor' kwarg
     response = await client_method("/", actor={"id": "root"})
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
 
 
 @pytest.mark.asyncio

tests/test_internals_request.py

@@ -1,7 +1,9 @@
-from datasette.utils.asgi import PayloadTooLarge, Request
 import json
+
 import pytest
 
+from datasette.utils.asgi import PayloadTooLarge, Request
+
 
 def _post_scope(headers=None):
     return {
@@ -33,6 +35,51 @@ def _receive_chunks(chunks):
     return receive
 
 
+@pytest.mark.parametrize(
+    "header_name", [b"content-type", b"Content-Type", b"CONTENT-TYPE"]
+)
+@pytest.mark.parametrize("lookup", ["content-type", "Content-Type", "CONTENT-TYPE"])
+def test_request_headers_case_insensitive(header_name, lookup):
+    request = Request({"headers": [(header_name, b"application/json")]}, None)
+    assert request.headers.get(lookup) == "application/json"
+    assert request.headers[lookup] == "application/json"
+    assert lookup in request.headers
+
+
+def test_request_headers_mapping():
+    request = Request(
+        {
+            "headers": [
+                (b"Content-Type", b"application/json"),
+                (b"X-Title", "café".encode("latin-1")),
+                (b"CONTENT-TYPE", b"text/plain"),
+            ]
+        },
+        None,
+    )
+    headers = request.headers
+    expected = {"content-type": "text/plain", "x-title": "café"}
+    assert headers == expected
+    assert dict(headers) == expected
+    assert list(headers) == list(expected)
+    assert list(headers.keys()) == list(expected.keys())
+    assert list(headers.items()) == list(expected.items())
+    assert json.loads(json.dumps(headers)) == expected
+    assert headers["Content-Type"] == "text/plain"
+    assert headers["X-Title"] == "café"
+
+
+@pytest.mark.parametrize("scope", [{}, {"headers": None}, {"headers": []}])
+def test_request_headers_missing(scope):
+    headers = Request(scope, None).headers
+    assert headers == {}
+    assert headers.get("Content-Type") is None
+    assert headers.get("Content-Type", "default") == "default"
+    assert "Content-Type" not in headers
+    with pytest.raises(KeyError):
+        headers["Content-Type"]
+
+
 @pytest.mark.asyncio
 async def test_request_post_vars():
     scope = {

tests/test_internals_response.py

@@ -1,7 +1,9 @@
-from datasette.utils.asgi import Response
 import json
+
 import pytest
 
+from datasette.utils.asgi import Response
+
 
 def test_response_html():
     response = Response.html("Hello from HTML")

tests/test_internals_urls.py

@@ -1,6 +1,7 @@
+import pytest
+
 from datasette.app import Datasette
 from datasette.utils import PrefixedUrlString
-import pytest
 
 
 @pytest.fixture(scope="module")

tests/test_label_column_for_table.py

@@ -1,6 +1,7 @@
 import pytest
-from datasette.database import Database
+
 from datasette.app import Datasette
+from datasette.database import Database
 
 
 @pytest.mark.asyncio

tests/test_lifespan.py

@@ -0,0 +1,344 @@
+"""
+Tests for wiring Datasette startup (setup_db table counts + invoke_startup)
+into the ASGI lifespan protocol.
+
+These exercise Datasette._startup_sequence() via three different callers:
+- AsgiLifespan, by hand-driving lifespan.startup messages (no HTTP request)
+- AsgiRunOnFirstRequest, the fallback for hosts that never send lifespan
+  events (this is what DatasetteClient / plain httpx2.ASGITransport uses)
+- Both at once, to prove startup hooks run at most once
+"""
+
+import asyncio
+import contextlib
+import sqlite3
+
+import httpx2
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.database import Database
+from datasette.plugins import pm
+
+
+async def _drive_lifespan_startup(app):
+    """Send a single lifespan.startup message into app's ASGI lifespan loop
+    and return the list of messages sent back - without ever sending
+    lifespan.shutdown. Mirrors what a real server does: after startup
+    completes it parks waiting for the next event. We cancel that wait
+    once we've observed the startup response, rather than closing the
+    Datasette instance down with a shutdown message.
+    """
+    messages_sent = []
+    startup_responded = asyncio.Event()
+    delivered = False
+
+    async def receive():
+        nonlocal delivered
+        if not delivered:
+            delivered = True
+            return {"type": "lifespan.startup"}
+        # No further messages: block until the task is cancelled below,
+        # same as a real server parked waiting for lifespan.shutdown.
+        await asyncio.Event().wait()
+
+    async def send(message):
+        messages_sent.append(message)
+        startup_responded.set()
+
+    task = asyncio.create_task(app({"type": "lifespan"}, receive, send))
+    try:
+        await asyncio.wait_for(startup_responded.wait(), timeout=5)
+    finally:
+        task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await task
+    return messages_sent
+
+
+@pytest.mark.asyncio
+async def test_lifespan_startup_runs_before_any_request():
+    ds = Datasette(memory=True)
+    assert ds._startup_invoked is False
+    app = ds.app()
+
+    messages = await _drive_lifespan_startup(app)
+
+    assert {"type": "lifespan.startup.complete"} in messages
+    assert ds._startup_invoked is True
+    # Internal catalog tables should be populated too, entirely without an
+    # HTTP request having been made.
+    internal_db = ds.get_internal_database()
+    databases = await internal_db.execute("select * from catalog_databases")
+    assert len(databases.rows) >= 1
+
+
+@pytest.mark.asyncio
+async def test_lifespan_startup_failure_reports_lifespan_startup_failed():
+    class RaisingStartupPlugin:
+        __name__ = "RaisingStartupPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                raise RuntimeError("boom from startup hook")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(RaisingStartupPlugin(), name="raising_startup_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan_startup(app)
+    finally:
+        pm.unregister(name="raising_startup_plugin")
+
+    assert messages == [
+        {"type": "lifespan.startup.failed", "message": "boom from startup hook"}
+    ]
+    # The exception happened before invoke_startup() got to the end of its
+    # body, so startup is not considered to have completed.
+    assert ds._startup_invoked is False
+
+
+@pytest.mark.asyncio
+async def test_startup_runs_exactly_once_across_lifespan_and_first_request():
+    call_count = {"n": 0}
+
+    class CountingStartupPlugin:
+        __name__ = "CountingStartupPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                call_count["n"] += 1
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(CountingStartupPlugin(), name="counting_startup_plugin")
+    try:
+        # Build the ASGI app once, the way a real deployment does - and
+        # reuse the SAME app instance for both the lifespan drive and the
+        # HTTP requests below, since a fresh ds.app() call would reset the
+        # AsgiRunOnFirstRequest fallback's state.
+        app = ds.app()
+
+        messages = await _drive_lifespan_startup(app)
+        assert {"type": "lifespan.startup.complete"} in messages
+        assert call_count["n"] == 1
+
+        # A first HTTP request (as if the host never sent lifespan events,
+        # or lifespan already ran) should not run the hook again.
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            response1 = await client.get("/-/versions.json")
+            assert response1.status_code == 200
+            # ... nor should a second, repeat request.
+            response2 = await client.get("/-/versions.json")
+            assert response2.status_code == 200
+    finally:
+        pm.unregister(name="counting_startup_plugin")
+
+    assert call_count["n"] == 1
+
+
+@pytest.mark.asyncio
+async def test_no_lifespan_first_request_still_triggers_startup():
+    # Pin today's behavior: a client that never drives ASGI lifespan events
+    # at all (like httpx2.ASGITransport, which DatasetteClient uses) still
+    # gets startup armed by the AsgiRunOnFirstRequest fallback.
+    ds = Datasette(memory=True)
+    assert ds._startup_invoked is False
+    app = ds.app()
+    transport = httpx2.ASGITransport(app=app)
+    async with httpx2.AsyncClient(
+        transport=transport, base_url="http://localhost"
+    ) as client:
+        response = await client.get("/-/versions.json")
+        assert response.status_code == 200
+
+    assert ds._startup_invoked is True
+    internal_db = ds.get_internal_database()
+    databases = await internal_db.execute("select * from catalog_databases")
+    assert len(databases.rows) >= 1
+
+
+@pytest.mark.asyncio
+async def test_datasette_client_first_request_triggers_startup():
+    # Same as above, but through the real DatasetteClient (ds.client) that
+    # plugins and tests actually use, to confirm nothing regressed there.
+    ds = Datasette(memory=True)
+    assert ds._startup_invoked is False
+    response = await ds.client.get("/-/versions.json")
+    assert response.status_code == 200
+    assert ds._startup_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_concurrent_first_requests_all_wait_for_slow_startup():
+    call_count = {"n": 0}
+
+    class SlowStartupPlugin:
+        __name__ = "SlowStartupPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                call_count["n"] += 1
+                await asyncio.sleep(0.2)
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(SlowStartupPlugin(), name="slow_startup_plugin")
+    try:
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            responses = await asyncio.gather(
+                *[client.get("/-/versions.json") for _ in range(10)]
+            )
+    finally:
+        pm.unregister(name="slow_startup_plugin")
+
+    # Every one of the 10 simultaneous first requests must have blocked
+    # until startup actually finished, not raced ahead of it.
+    assert all(response.status_code == 200 for response in responses)
+    assert call_count["n"] == 1
+    assert ds._startup_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_setup_db_still_runs_when_invoke_startup_ran_first(tmp_path, monkeypatch):
+    # Regression test: `datasette serve` (cli.py _serve_async) calls
+    # ds.invoke_startup() directly, before uvicorn ever sends a
+    # lifespan.startup event that drives _startup_sequence(). If
+    # _startup_sequence()'s fast path only checked `_startup_invoked`, it
+    # would see startup already done and skip the immutable-database
+    # table-count precompute (setup_db) entirely - a silent regression
+    # versus main, where AsgiRunOnFirstRequest ran setup_db unconditionally
+    # on request #1.
+    db_path = tmp_path / "immutable.db"
+    conn = sqlite3.connect(str(db_path))
+    conn.execute("create table t (id integer primary key)")
+    conn.commit()
+    conn.close()
+
+    ds = Datasette([], immutables=[str(db_path)])
+
+    call_count = {"n": 0}
+    original_table_counts = Database.table_counts
+
+    async def counting_table_counts(self, *args, **kwargs):
+        call_count["n"] += 1
+        return await original_table_counts(self, *args, **kwargs)
+
+    monkeypatch.setattr(Database, "table_counts", counting_table_counts)
+
+    # Simulate the CLI path: invoke_startup() runs directly and completes
+    # BEFORE _startup_sequence() ever gets a chance to run setup_db.
+    await ds.invoke_startup()
+    assert ds._startup_invoked is True
+    assert call_count["n"] == 0
+
+    # The lifespan/first-request path (or the CLI itself, per the fix)
+    # calling the shared entry point afterwards must still precompute
+    # table counts for immutable databases.
+    await ds._startup_sequence()
+    assert call_count["n"] == 1
+    assert ds._setup_db_done is True
+
+    # Idempotency: a second call must not recompute.
+    await ds._startup_sequence()
+    assert call_count["n"] == 1
+
+
+@pytest.mark.asyncio
+async def test_asgi_wrapper_runs_after_startup_fallback_path():
+    class AssertStartupPlugin:
+        __name__ = "AssertStartupPlugin"
+
+        @hookimpl
+        def asgi_wrapper(self, datasette):
+            def wrap(app):
+                async def check_startup(scope, receive, send):
+                    if scope["type"] == "http":
+                        assert (
+                            datasette._startup_invoked is True
+                        ), "asgi_wrapper saw an http scope before startup completed"
+                    await app(scope, receive, send)
+
+                return check_startup
+
+            return wrap
+
+    ds = Datasette(memory=True)
+    pm.register(AssertStartupPlugin(), name="assert_startup_plugin")
+    try:
+        assert ds._startup_invoked is False
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            response = await client.get("/-/versions.json")
+            assert response.status_code == 200
+    finally:
+        pm.unregister(name="assert_startup_plugin")
+
+    assert ds._startup_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_short_circuit_wrapper_no_longer_defers_startup():
+    # Middleware that returns a response before getting to the rest of
+    # Datasette should still cause _startup_invoked=True
+    class ShortCircuitPlugin:
+        __name__ = "ShortCircuitPlugin"
+
+        @hookimpl
+        def asgi_wrapper(self, datasette):
+            def wrap(app):
+                async def forbidden(scope, receive, send):
+                    if scope["type"] != "http":
+                        await app(scope, receive, send)
+                        return
+                    await send(
+                        {
+                            "type": "http.response.start",
+                            "status": 403,
+                            "headers": [[b"content-type", b"text/plain"]],
+                        }
+                    )
+                    await send(
+                        {
+                            "type": "http.response.body",
+                            "body": b"Forbidden",
+                        }
+                    )
+
+                return forbidden
+
+            return wrap
+
+    ds = Datasette(memory=True)
+    pm.register(ShortCircuitPlugin(), name="short_circuit_plugin")
+    try:
+        assert ds._startup_invoked is False
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            response = await client.get("/-/versions.json")
+            assert response.status_code == 403
+    finally:
+        pm.unregister(name="short_circuit_plugin")
+
+    assert ds._startup_invoked is True

tests/test_load_extensions.py

@@ -1,6 +1,9 @@
-from datasette.app import Datasette
-import pytest
 from pathlib import Path
+from unittest import mock
+
+import pytest
+
+from datasette.app import Datasette
 
 # not necessarily a full path - the full compiled path looks like "ext.dylib"
 # or another suffix, but sqlite will, under the hood, decide which file
@@ -18,6 +21,29 @@ def has_compiled_ext():
     return False
 
 
+@pytest.mark.parametrize("load_fails", (False, True))
+def test_load_extension_is_disabled(load_fails):
+    ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
+    connection = mock.Mock()
+    if load_fails:
+        connection.load_extension.side_effect = RuntimeError
+
+    if load_fails:
+        with pytest.raises(RuntimeError):
+            ds._prepare_connection(connection, "data")
+    else:
+        ds._prepare_connection(connection, "data")
+
+    # Extensions are loaded using the Python API, never via SQL
+    assert connection.load_extension.mock_calls == [
+        mock.call(COMPILED_EXTENSION_PATH),
+    ]
+    assert connection.enable_load_extension.mock_calls == [
+        mock.call(True),
+        mock.call(False),
+    ]
+
+
 @pytest.mark.asyncio
 @pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
 async def test_load_extension_default_entrypoint():
@@ -62,3 +88,20 @@ async def test_load_extension_multiple_entrypoints():
     response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+c()")
     assert response.status_code == 200
     assert response.json()["rows"][0][0] == "c"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
+async def test_sql_cannot_load_additional_extension():
+    ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
+
+    response = await ds.client.get(
+        "/_memory/-/query.json",
+        params={
+            "sql": "select load_extension(:path, :entrypoint)",
+            "path": COMPILED_EXTENSION_PATH,
+            "entrypoint": "sqlite3_ext_b_init",
+        },
+    )
+    assert response.status_code == 400
+    assert response.json()["error"] == "not authorized"

tests/test_messages.py

@@ -1,6 +1,7 @@
-from .utils import cookie_was_deleted
 import pytest
 
+from .utils import cookie_was_deleted
+
 
 @pytest.mark.asyncio
 @pytest.mark.parametrize(

tests/test_multipart.py

@@ -4,14 +4,145 @@ Tests for request.form() multipart form data parsing.
 Uses TDD approach - these tests are written first, then implementation follows.
 """
 
+import asyncio
 import base64
 import json
-import pytest
+import threading
 from collections import namedtuple
 
+import pytest
 from multipart_form_data_conformance import get_tests_dir
 
-from datasette.utils.asgi import Request, BadRequest
+from datasette.utils.asgi import BadRequest, Request
+from datasette.utils.multipart import MultipartParseError, parse_form_data
+
+
+@pytest.fixture
+def upload_files(monkeypatch):
+    from datasette.utils import multipart
+
+    files = []
+    original = multipart.tempfile.SpooledTemporaryFile
+
+    def create_file(*args, **kwargs):
+        file = original(*args, **kwargs)
+        files.append(file)
+        return file
+
+    monkeypatch.setattr(multipart.tempfile, "SpooledTemporaryFile", create_file)
+    yield files
+    for file in files:
+        file.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "failure",
+    [
+        "file_limit",
+        "request_limit",
+        "truncated",
+        "disconnect",
+        "receive_error",
+        "cancel",
+    ],
+)
+async def test_failed_upload_closes_completed_and_partial_files(upload_files, failure):
+    # Complete one file and begin another, flushing the parser's 64 KiB batch.
+    body = (
+        b'--boundary\r\nContent-Disposition: form-data; name="one"; filename="one"\r\n\r\n'
+        b'first\r\n--boundary\r\nContent-Disposition: form-data; name="two"; filename="two"\r\n\r\n'
+        + b"x" * (64 * 1024)
+    )
+    calls = 0
+
+    async def receive():
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            return {"type": "http.request", "body": body, "more_body": True}
+        if failure == "disconnect":
+            return {"type": "http.disconnect"}
+        if failure == "receive_error":
+            raise OSError("receive failed")
+        if failure == "cancel":
+            raise asyncio.CancelledError
+        return {"type": "http.request", "body": b"x" * 100, "more_body": False}
+
+    kwargs = {}
+    if failure == "file_limit":
+        kwargs["max_file_size"] = 1024
+    if failure == "request_limit":
+        kwargs["max_request_size"] = len(body)
+    error = {
+        "receive_error": OSError,
+        "cancel": asyncio.CancelledError,
+    }.get(failure, MultipartParseError)
+    with pytest.raises(error):
+        await parse_form_data(
+            receive, "multipart/form-data; boundary=boundary", files=True, **kwargs
+        )
+    assert len(upload_files) == 2
+    assert all(file.closed for file in upload_files)
+
+
+@pytest.mark.asyncio
+async def test_unnamed_upload_is_closed(upload_files):
+    body = (
+        b'--boundary\r\nContent-Disposition: form-data; filename="ignored"\r\n\r\n'
+        b"content\r\n--boundary--\r\n"
+    )
+    form = await parse_form_data(
+        make_receive(body), "multipart/form-data; boundary=boundary", files=True
+    )
+    assert len(form) == 0
+    assert len(upload_files) == 1
+    assert upload_files[0].closed
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("worker_error", [False, True])
+async def test_cancelled_upload_waits_for_worker(
+    upload_files, monkeypatch, worker_error
+):
+    from datasette.utils.multipart import MultipartParser
+
+    loop = asyncio.get_running_loop()
+    started = asyncio.Event()
+    release = threading.Event()
+    original_feed = MultipartParser.feed
+
+    def blocking_feed(self, chunk):
+        original_feed(self, chunk)
+        loop.call_soon_threadsafe(started.set)
+        assert release.wait(5)
+        if worker_error:
+            raise MultipartParseError("worker failed")
+
+    monkeypatch.setattr(MultipartParser, "feed", blocking_feed)
+    body = (
+        b'--boundary\r\nContent-Disposition: form-data; name="file"; filename="file"\r\n\r\n'
+        + b"x" * (64 * 1024)
+    )
+    task = asyncio.create_task(
+        parse_form_data(
+            make_receive(body), "multipart/form-data; boundary=boundary", files=True
+        )
+    )
+    try:
+        await asyncio.wait_for(started.wait(), timeout=5)
+        # A second cancellation must also leave cleanup waiting for the worker.
+        for _ in range(2):
+            task.cancel()
+            await asyncio.sleep(0)
+        assert not task.done()
+        assert len(upload_files) == 1
+        assert not upload_files[0].closed
+    finally:
+        release.set()
+        with pytest.raises(asyncio.CancelledError):
+            await task
+    assert upload_files[0].closed
 
 
 def make_receive(body: bytes):
@@ -1078,75 +1209,73 @@ async def test_conformance(test_spec, headers, body):
             await request.form(files=True)
         return
 
-    # Parse form data
-    form = await request.form(files=True)
+    async with await request.form(files=True) as form:
+        # Verify each expected part
+        for i, expected_part in enumerate(expected["parts"]):
+            name = expected_part["name"]
 
-    # Verify each expected part
-    for i, expected_part in enumerate(expected["parts"]):
-        name = expected_part["name"]
+            # Get value(s) for this name
+            values = form.getlist(name)
 
-        # Get value(s) for this name
-        values = form.getlist(name)
+            # Find the value at the correct index for this name
+            # (handles multiple values with same name)
+            same_name_count = sum(1 for p in expected["parts"][:i] if p["name"] == name)
 
-        # Find the value at the correct index for this name
-        # (handles multiple values with same name)
-        same_name_count = sum(1 for p in expected["parts"][:i] if p["name"] == name)
+            if same_name_count >= len(values):
+                pytest.fail(
+                    f"Expected part {name} at index {same_name_count} but only {len(values)} found"
+                )
 
-        if same_name_count >= len(values):
-            pytest.fail(
-                f"Expected part {name} at index {same_name_count} but only {len(values)} found"
+            value = values[same_name_count]
+
+            # Determine expected content
+            if "body_base64" in expected_part:
+                expected_content = base64.b64decode(expected_part["body_base64"])
+            elif "body_text" in expected_part:
+                expected_content = expected_part["body_text"].encode("utf-8")
+            else:
+                expected_content = None
+
+            # Check for file vs field
+            # A part is a file if it has a filename OR filename_star
+            is_file = (
+                expected_part.get("filename") is not None
+                or expected_part.get("filename_star") is not None
             )
 
-        value = values[same_name_count]
+            if is_file:
+                # It's a file
+                assert hasattr(value, "filename"), f"Expected file for {name}"
 
-        # Determine expected content
-        if "body_base64" in expected_part:
-            expected_content = base64.b64decode(expected_part["body_base64"])
-        elif "body_text" in expected_part:
-            expected_content = expected_part["body_text"].encode("utf-8")
-        else:
-            expected_content = None
+                # Check filename - use filename_star if present, else filename
+                expected_filename = expected_part.get(
+                    "filename_star"
+                ) or expected_part.get("filename")
+                if expected_filename:
+                    assert (
+                        value.filename == expected_filename
+                    ), f"Filename mismatch: expected {expected_filename!r}, got {value.filename!r}"
 
-        # Check for file vs field
-        # A part is a file if it has a filename OR filename_star
-        is_file = (
-            expected_part.get("filename") is not None
-            or expected_part.get("filename_star") is not None
-        )
+                if expected_part.get("content_type"):
+                    assert value.content_type == expected_part["content_type"]
 
-        if is_file:
-            # It's a file
-            assert hasattr(value, "filename"), f"Expected file for {name}"
-
-            # Check filename - use filename_star if present, else filename
-            expected_filename = expected_part.get("filename_star") or expected_part.get(
-                "filename"
-            )
-            if expected_filename:
+                content = await value.read()
                 assert (
-                    value.filename == expected_filename
-                ), f"Filename mismatch: expected {expected_filename!r}, got {value.filename!r}"
+                    len(content) == expected_part["body_size"]
+                ), f"Size mismatch: expected {expected_part['body_size']}, got {len(content)}"
+                if expected_content is not None:
+                    assert content == expected_content
+            else:
+                # It's a text field
+                if hasattr(value, "filename"):
+                    pytest.fail(f"Expected text field for {name}, got file")
 
-            if expected_part.get("content_type"):
-                assert value.content_type == expected_part["content_type"]
-
-            content = await value.read()
-            assert (
-                len(content) == expected_part["body_size"]
-            ), f"Size mismatch: expected {expected_part['body_size']}, got {len(content)}"
-            if expected_content is not None:
-                assert content == expected_content
-        else:
-            # It's a text field
-            if hasattr(value, "filename"):
-                pytest.fail(f"Expected text field for {name}, got file")
-
-            if expected_content is not None:
-                # For text fields, value is a string
-                try:
-                    expected_text = expected_content.decode("utf-8")
-                except UnicodeDecodeError:
-                    expected_text = expected_content.decode("latin-1")
-                assert (
-                    value == expected_text
-                ), f"Value mismatch: expected {expected_text!r}, got {value!r}"
+                if expected_content is not None:
+                    # For text fields, value is a string
+                    try:
+                        expected_text = expected_content.decode("utf-8")
+                    except UnicodeDecodeError:
+                        expected_text = expected_content.decode("latin-1")
+                    assert (
+                        value == expected_text
+                    ), f"Value mismatch: expected {expected_text!r}, got {value!r}"

tests/test_numeric_filter_values.py

@@ -0,0 +1,47 @@
+import sqlite3
+from contextlib import closing
+
+import pytest
+
+from datasette.filters import Filters
+
+
+@pytest.mark.parametrize(
+    "value,expected",
+    (
+        ("3.5", 3.5),
+        ("-2", -2),
+        ("-2.5", -2.5),
+        ("1e3", 1000.0),
+        ("not-a-number", "not-a-number"),
+        ("nan", "nan"),
+        ("inf", "inf"),
+        ("-inf", "-inf"),
+    ),
+)
+def test_numeric_filter_parameters(value, expected):
+    filters = Filters((("score__gt", value),))
+    sql_bits, params = filters.build_where_clauses("items")
+
+    assert sql_bits == ['"score" > :p0']
+    assert params == {"p0": expected}
+
+
+def test_numeric_filter_parameters_against_calculated_view():
+    with closing(sqlite3.connect(":memory:")) as conn:
+        conn.execute("create table searchable(pk integer)")
+        conn.executemany("insert into searchable(pk) values (?)", [(0,), (1,), (2,)])
+        conn.execute(
+            "create view calculated as "
+            "select pk + 1 as pk_plus_one, pk / 2.0 as score from searchable"
+        )
+
+        sql_bits, params = Filters((("score__gt", "0.1"),)).build_where_clauses(
+            "calculated"
+        )
+        rows = conn.execute(
+            "select score from calculated where {}".format(" and ".join(sql_bits)),
+            params,
+        ).fetchall()
+
+        assert rows == [(0.5,), (1.0,)]

tests/test_package.py

@@ -1,9 +1,11 @@
-from click.testing import CliRunner
-from datasette import cli
-from unittest import mock
 import os
 import pathlib
+from unittest import mock
+
 import pytest
+from click.testing import CliRunner
+
+from datasette import cli
 
 
 class CaptureDockerfile:

tests/test_permission_endpoints.py

@@ -6,6 +6,7 @@ Tests for permission endpoints:
 
 import pytest
 import pytest_asyncio
+
 from datasette.app import Datasette
 
 
@@ -432,8 +433,8 @@ async def test_execute_sql_requires_view_database():
     A user who has execute-sql permission but not view-database permission should not
     be able to execute SQL on that database.
     """
-    from datasette.permissions import PermissionSQL
     from datasette import hookimpl
+    from datasette.permissions import PermissionSQL
 
     class TestPermissionPlugin:
         __name__ = "TestPermissionPlugin"
@@ -493,3 +494,31 @@ async def test_execute_sql_requires_view_database():
         )
     finally:
         ds.pm.unregister(plugin)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("path", ["/-/allowed", "/-/allowed.json?action=view-table"])
+async def test_allowed_requires_view_instance(path):
+    """
+    GHSA-hp2x-vx2r-6vxg: /-/allowed should be gated like its /-/rules sibling.
+
+    An actor who is denied view-instance gets 403 from / and /-/rules, but
+    /-/allowed (HTML and JSON) currently returns 200 to the same actor.
+    """
+    ds = Datasette(config={"allow": {"id": "alice"}})
+    await ds.invoke_startup()
+    db = ds.add_memory_database("live")
+    await db.execute_write("CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY)")
+    await ds.refresh_schemas()
+
+    assert (await ds.client.get("/")).status_code == 403
+    assert (await ds.client.get("/-/rules.json?action=view-table")).status_code == 403
+
+    response = await ds.client.get(path)
+    assert response.status_code == 403
+
+    # Alice is still allowed
+    response = await ds.client.get(
+        path, cookies={"ds_actor": ds.client.actor_cookie({"id": "alice"})}
+    )
+    assert response.status_code == 200

tests/test_permissions.py

@@ -1,20 +1,23 @@
 import collections
+import copy
+import json
+import re
+import time
+import urllib
+from pprint import pprint
+
+import pytest
+import pytest_asyncio
 from asgiref.sync import async_to_sync
+from bs4 import BeautifulSoup as Soup
+from click.testing import CliRunner
+
 from datasette.app import Datasette
 from datasette.cli import cli
 from datasette.default_permissions import restrictions_allow_action
 from datasette.utils import UNSTABLE_API_MESSAGE
+
 from .fixtures import assert_permissions_checked, make_app_client
-from click.testing import CliRunner
-from bs4 import BeautifulSoup as Soup
-import copy
-import json
-from pprint import pprint
-import pytest_asyncio
-import pytest
-import re
-import time
-import urllib
 
 
 @pytest.fixture(scope="module")
@@ -457,6 +460,20 @@ async def test_permissions_debug(ds_client, filter_):
     assert checks == expected_checks
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "permissions_debug,expected_status",
+    (
+        (1, 200),
+        (0, 403),
+    ),
+)
+async def test_permissions_debug_numeric_boolean(permissions_debug, expected_status):
+    ds = Datasette(config={"permissions": {"permissions-debug": permissions_debug}})
+    response = await ds.client.get("/-/permissions")
+    assert response.status_code == expected_status
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "actor,allow,expected_fragment",
@@ -503,15 +520,22 @@ def view_instance_client():
         "/-/plugins",
         "/-/settings",
         "/-/threads",
+        "/-/tasks",
         "/-/databases",
         "/-/permissions",
         "/-/messages",
         "/-/patterns",
+        "/-/patterns/menus",
     ],
 )
 def test_view_instance(path, view_instance_client):
     assert 403 == view_instance_client.get(path).status
-    if path not in ("/-/permissions", "/-/messages", "/-/patterns"):
+    if path not in (
+        "/-/permissions",
+        "/-/messages",
+        "/-/patterns",
+        "/-/patterns/menus",
+    ):
         assert 403 == view_instance_client.get(path + ".json").status
 
 
@@ -588,9 +612,7 @@ def test_permissions_cascade(cascade_app_client, path, permissions, expected_sta
         )
         assert (
             response.status == expected_status
-        ), "path: {}, permissions: {}, expected_status: {}, status: {}".format(
-            path, permissions, expected_status, response.status
-        )
+        ), f"path: {path}, permissions: {permissions}, expected_status: {expected_status}, status: {response.status}"
     finally:
         cascade_app_client.ds.config = previous_config
 
@@ -748,7 +770,12 @@ async def test_actor_restricted_permissions(
     }
     if actor.get("id"):
         expected["actor_id"] = actor["id"]
-    assert response.json() == expected
+    data = response.json()
+    for key, value in expected.items():
+        assert data[key] == value
+    assert data["actor"] == actor
+    assert data["explanation"]["allowed"] is expected_result
+    assert data["explanation"]["summary"]
 
 
 PermConfigTestCase = collections.namedtuple(
@@ -1734,6 +1761,8 @@ async def test_permission_check_view_requires_debug_permission():
     data = response.json()
     assert data["action"] == "view-instance"
     assert data["allowed"] is True
+    assert data["explanation"]["allowed"] is True
+    assert data["explanation"]["summary"]
 
 
 @pytest.mark.asyncio
@@ -1759,6 +1788,211 @@ async def test_permission_check_view_query_actions(action):
     }
 
 
+@pytest.mark.asyncio
+async def test_permission_check_explains_specificity_for_hypothetical_actor():
+    ds = Datasette(
+        config={
+            "permissions": {"view-table": {"id": "alice"}},
+            "databases": {
+                "analytics": {
+                    "permissions": {"view-table": False},
+                    "tables": {
+                        "public": {"permissions": {"view-table": {"id": "alice"}}}
+                    },
+                }
+            },
+        }
+    )
+    ds.root_enabled = True
+    await ds.invoke_startup()
+
+    def path_for(child):
+        return "/-/check.json?" + urllib.parse.urlencode(
+            {
+                "action": "view-table",
+                "parent": "analytics",
+                "child": child,
+                "actor": json.dumps({"id": "alice"}),
+            }
+        )
+
+    public_response = await ds.client.get(path_for("public"), actor={"id": "root"})
+    assert public_response.status_code == 200
+    public = public_response.json()
+    assert public["actor"] == {"id": "alice"}
+    assert public["allowed"] is True
+    assert public["explanation"]["allowed"] is True
+    assert public["explanation"]["winning_scope"] == "resource"
+    public_rules = public["explanation"]["matched_rules"]
+    assert any(
+        rule["scope"] == "resource" and rule["effect"] == "allow" and rule["decisive"]
+        for rule in public_rules
+    )
+    assert any(
+        rule["scope"] == "parent"
+        and rule["effect"] == "deny"
+        and rule["ignored_because"] == "A more specific rule matched"
+        for rule in public_rules
+    )
+
+    private_response = await ds.client.get(path_for("private"), actor={"id": "root"})
+    assert private_response.status_code == 200
+    private = private_response.json()
+    assert private["allowed"] is False
+    assert private["explanation"]["allowed"] is False
+    assert private["explanation"]["winning_scope"] == "parent"
+    assert private["explanation"]["summary"].startswith("Denied by a parent-level rule")
+
+
+@pytest.mark.asyncio
+async def test_permission_check_explains_deny_wins_at_same_scope():
+    ds = Datasette(config={"permissions": {"view-table": {"id": "someone-else"}}})
+    ds.root_enabled = True
+    await ds.invoke_startup()
+    path = "/-/check.json?" + urllib.parse.urlencode(
+        {
+            "action": "view-table",
+            "parent": "analytics",
+            "child": "users",
+            "actor": json.dumps({"id": "alice"}),
+        }
+    )
+    response = await ds.client.get(path, actor={"id": "root"})
+    assert response.status_code == 200
+    data = response.json()
+    assert data["allowed"] is False
+    assert data["explanation"]["winning_scope"] == "global"
+    rules = data["explanation"]["matched_rules"]
+    assert any(rule["effect"] == "deny" and rule["decisive"] for rule in rules)
+    assert any(
+        rule["effect"] == "allow"
+        and rule["ignored_because"] == "A deny rule matched at the same scope"
+        for rule in rules
+    )
+
+
+@pytest.mark.asyncio
+async def test_permission_check_explains_default_deny():
+    ds = Datasette()
+    ds.root_enabled = True
+    await ds.invoke_startup()
+    path = "/-/check.json?" + urllib.parse.urlencode(
+        {
+            "action": "insert-row",
+            "parent": "analytics",
+            "child": "users",
+            "actor": json.dumps({"id": "alice"}),
+        }
+    )
+    response = await ds.client.get(path, actor={"id": "root"})
+    assert response.status_code == 200
+    data = response.json()
+    assert data["allowed"] is False
+    explanation = data["explanation"]
+    assert explanation["allowed"] is False
+    assert explanation["matched_rules"] == []
+    assert explanation["winning_scope"] is None
+    assert explanation["summary"] == (
+        "Denied because no permission rule matched this actor and resource."
+    )
+
+
+@pytest.mark.asyncio
+async def test_permission_check_explains_actor_restrictions():
+    ds = Datasette()
+    ds.root_enabled = True
+    await ds.invoke_startup()
+    restricted_actor = {
+        "id": "alice",
+        "_r": {"r": {"analytics": {"public": ["vt"]}}},
+    }
+    path = "/-/check.json?" + urllib.parse.urlencode(
+        {
+            "action": "view-table",
+            "parent": "analytics",
+            "child": "private",
+            "actor": json.dumps(restricted_actor),
+        }
+    )
+    response = await ds.client.get(path, actor={"id": "root"})
+    assert response.status_code == 200
+    data = response.json()
+    assert data["allowed"] is False
+    explanation = data["explanation"]
+    assert explanation["rule_allowed"] is True
+    assert explanation["restriction_allowed"] is False
+    assert explanation["allowed"] is False
+    assert explanation["restrictions"]
+    assert any(
+        restriction["allowed"] is False for restriction in explanation["restrictions"]
+    )
+    assert "actor's restrictions" in explanation["summary"]
+
+
+@pytest.mark.asyncio
+async def test_permission_check_explains_required_actions():
+    from datasette import hookimpl
+    from datasette.permissions import PermissionSQL
+
+    class StoreQueryPermissions:
+        @hookimpl
+        def permission_resources_sql(self, actor, action):
+            if not actor or actor.get("id") != "alice":
+                return None
+            if action == "store-query":
+                return PermissionSQL(
+                    sql="SELECT 'analytics' AS parent, NULL AS child, 1 AS allow, 'alice can store queries' AS reason"
+                )
+            if action == "execute-sql":
+                return PermissionSQL(
+                    sql="SELECT 'analytics' AS parent, NULL AS child, 0 AS allow, 'alice cannot execute SQL' AS reason"
+                )
+
+    ds = Datasette()
+    ds.root_enabled = True
+    await ds.invoke_startup()
+    ds.pm.register(StoreQueryPermissions(), name="store-query-test")
+    path = "/-/check.json?" + urllib.parse.urlencode(
+        {
+            "action": "store-query",
+            "parent": "analytics",
+            "actor": json.dumps({"id": "alice"}),
+        }
+    )
+    response = await ds.client.get(path, actor={"id": "root"})
+    assert response.status_code == 200
+    data = response.json()
+    assert data["allowed"] is False
+    explanation = data["explanation"]
+    assert explanation["rule_allowed"] is True
+    assert explanation["required_actions"][0]["action"] == "execute-sql"
+    assert explanation["required_actions"][0]["allowed"] is False
+    assert explanation["summary"] == (
+        "Denied because store-query also requires execute-sql, which was denied."
+    )
+
+
+@pytest.mark.asyncio
+async def test_permission_check_hypothetical_actor_validation():
+    ds = Datasette()
+    ds.root_enabled = True
+    await ds.invoke_startup()
+
+    response = await ds.client.get(
+        "/-/check.json?action=view-instance&actor=not-json",
+        actor={"id": "root"},
+    )
+    assert response.status_code == 400
+    assert response.json()["error"].startswith("Invalid actor JSON:")
+
+    response = await ds.client.get(
+        "/-/check.json?action=view-instance&actor=%5B%5D",
+        actor={"id": "root"},
+    )
+    assert response.status_code == 400
+    assert response.json()["error"] == "actor must be a JSON object or null"
+
+
 @pytest.mark.asyncio
 async def test_root_allow_block_with_table_restricted_actor():
     """
@@ -1813,7 +2047,7 @@ async def test_databases_json_respects_view_database(tmp_path_factory):
 
     paths = []
     for name in ("public", "private"):
-        path = str(db_directory / "{}.db".format(name))
+        path = str(db_directory / f"{name}.db")
         conn = _sqlite3.connect(path)
         conn.execute("vacuum")
         conn.close()

tests/test_playwright.py

@@ -5,7 +5,7 @@ import subprocess
 import sys
 import time
 
-import httpx
+import httpx2
 import pytest
 
 from datasette.fixtures import write_fixture_database
@@ -34,11 +34,11 @@ def wait_for_server(process, url, timeout=30):
                 f"stderr:\n{stderr}"
             )
         try:
-            response = httpx.get(url, timeout=1.0)
+            response = httpx2.get(url, timeout=1.0)
             if response.status_code < 500:
                 return
             last_error = f"HTTP {response.status_code}: {response.text[:200]}"
-        except httpx.HTTPError as ex:
+        except httpx2.HTTPError as ex:
             last_error = repr(ex)
         time.sleep(0.1)
     if process.poll() is None:
@@ -108,6 +108,11 @@ def write_playwright_database(db_path):
     conn = sqlite3.connect(db_path)
     try:
         conn.executescript("""
+        create table count_numbers (id integer primary key);
+        with recursive sequence(id) as (
+            select 1 union all select id + 1 from sequence where id < 10002
+        )
+        insert into count_numbers select id from sequence;
         create table projects (
             id integer primary key,
             title text not null,
@@ -336,7 +341,7 @@ def project_rows(datasette_server, **filters):
         "_shape": "objects",
         **{key: str(value) for key, value in filters.items()},
     }
-    response = httpx.get(f"{datasette_server}data/projects.json", params=params)
+    response = httpx2.get(f"{datasette_server}data/projects.json", params=params)
     response.raise_for_status()
     return response.json()["rows"]
 
@@ -348,7 +353,7 @@ def project_row(datasette_server, pk):
 
 
 def binary_file_blob(datasette_server, pk):
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/binary_files/{pk}.blob",
         params={"_blob_column": "data"},
     )
@@ -369,7 +374,7 @@ def bulk_default_rows(datasette_server, **filters):
         "_shape": "objects",
         **{key: str(value) for key, value in filters.items()},
     }
-    response = httpx.get(f"{datasette_server}data/bulk_defaults.json", params=params)
+    response = httpx2.get(f"{datasette_server}data/bulk_defaults.json", params=params)
     response.raise_for_status()
     return response.json()["rows"]
 
@@ -379,7 +384,7 @@ def upsert_item_rows(datasette_server, **filters):
         "_shape": "objects",
         **{key: str(value) for key, value in filters.items()},
     }
-    response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params)
+    response = httpx2.get(f"{datasette_server}data/upsert_items.json", params=params)
     response.raise_for_status()
     return response.json()["rows"]
 
@@ -473,7 +478,7 @@ def test_create_table_flow(page, datasette_server):
     page.wait_for_url("**/data/playwright_created")
     assert "playwright_created" in page.locator("h1").inner_text()
 
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/playwright_created.json?_extra=columns,column_types"
     )
     response.raise_for_status()
@@ -487,7 +492,7 @@ def test_create_table_flow(page, datasette_server):
     assert data["column_types"] == {
         "metadata": {"type": "json", "config": None},
     }
-    schema_response = httpx.get(
+    schema_response = httpx2.get(
         f"{datasette_server}data/-/query.json",
         params={
             "sql": (
@@ -603,7 +608,7 @@ def test_create_table_from_data_flow(page, datasette_server):
     dialog.locator(".table-create-save").click()
     page.wait_for_url("**/data/playwright_from_data")
 
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/playwright_from_data.json?_shape=objects"
     )
     response.raise_for_status()
@@ -639,7 +644,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
     dialog.locator(".table-create-save").click()
     page.wait_for_url("**/data/playwright_numeric_blanks")
 
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects"
     )
     response.raise_for_status()
@@ -648,7 +653,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
         {"name": "B", "score": None},
     ]
 
-    schema_response = httpx.get(
+    schema_response = httpx2.get(
         f"{datasette_server}data/-/query.json",
         params={
             "sql": (
@@ -856,7 +861,7 @@ def test_alter_table_flow(page, datasette_server):
 
     columns = []
     for _ in range(20):
-        response = httpx.get(f"{datasette_server}data/projects.json?_extra=columns")
+        response = httpx2.get(f"{datasette_server}data/projects.json?_extra=columns")
         response.raise_for_status()
         columns = response.json()["columns"]
         if "status" in columns:
@@ -1028,15 +1033,14 @@ def test_alter_table_cancel_skips_discard_prompt(page, datasette_server):
     dialog.locator(".table-alter-add-column").click()
     dialog.locator(".table-alter-column-name").last.fill("escape_me")
     page.keyboard.press("Escape")
+    page.wait_for_function("window.__discardConfirmMessages.length === 1")
     assert page.evaluate("() => window.__discardConfirmMessages") == [
         "Discard table changes?"
     ]
     assert dialog.evaluate("node => node.open") is True
 
     page.evaluate("() => window.__discardConfirmMessages = []")
-    dialog.evaluate(
-        """node => node.dispatchEvent(new MouseEvent("click", {bubbles: true}))"""
-    )
+    page.mouse.click(2, 2)
     assert page.evaluate("() => window.__discardConfirmMessages") == [
         "Discard table changes?"
     ]
@@ -1079,6 +1083,92 @@ def test_navigation_search_renders_jump_sections_from_javascript_plugins(
     page.wait_for_url("**/-/playwright-agent")
 
 
+@pytest.mark.playwright
+def test_navigation_search_created_from_javascript(page, datasette_server):
+    from playwright.sync_api import expect
+
+    page.goto(datasette_server)
+    page.evaluate("""() => {
+        const search = document.createElement('navigation-search');
+        search.id = 'additional-search';
+        search.setAttribute('items', JSON.stringify([
+            {name: 'Projects', url: '/data/projects'}
+        ]));
+        document.body.append(search);
+        const unrelated = document.createElement('div');
+        unrelated.className = 'search-container';
+        unrelated.id = 'outside-search';
+        document.body.append(unrelated);
+        search.openMenu();
+    }""")
+    search = page.locator("#additional-search")
+    dialog = search.get_by_role("dialog", name="Jump to", exact=True)
+    expect(dialog).to_be_visible()
+    # Page styles and ordinary DOM queries can reach the component's controls.
+    page.add_style_tag(
+        content="#additional-search .search-input { border-top-color: rgb(1, 2, 3); }"
+    )
+    field = dialog.get_by_role("combobox", name="Jump to", exact=True)
+    expect(field).to_have_css("border-top-color", "rgb(1, 2, 3)")
+    assert field.evaluate("node => document.getElementById(node.id) === node")
+    expect(page.locator("#outside-search")).to_have_css("display", "block")
+    field.fill("projects")
+    expect(dialog.get_by_role("option")).to_contain_text("Projects")
+    field.press("Enter")
+    page.wait_for_url("**/data/projects")
+
+
+@pytest.mark.playwright
+def test_column_chooser_selection_and_drag_in_document(page, datasette_server):
+    from playwright.sync_api import expect
+
+    page.goto(datasette_server + "data/projects")
+    page.emulate_media(reduced_motion="reduce")
+    page.evaluate("""() => {
+        const chooser = document.createElement('column-chooser');
+        chooser.id = 'additional-chooser';
+        document.body.append(chooser);
+        window.appliedColumns = null;
+        chooser.open({
+            columns: ['title', 'notes', 'score'],
+            selected: ['title', 'notes'],
+            onApply: columns => { window.appliedColumns = columns; }
+        });
+    }""")
+    chooser = page.locator("#additional-chooser")
+    dialog = chooser.get_by_role("dialog", name="Choose columns")
+    expect(dialog).to_be_visible()
+    assert dialog.evaluate("""node => {
+        const id = node.getAttribute('aria-labelledby');
+        return document.querySelectorAll(`#${id}`).length === 1 &&
+            node.contains(document.getElementById(id));
+    }""")
+    expect(dialog.locator(".modal-meta")).to_have_text("2 of 3 selected")
+    dialog.get_by_role("button", name="Deselect all", exact=True).click()
+    expect(dialog.locator(".modal-meta")).to_have_text("0 of 3 selected")
+    dialog.get_by_role("button", name="Select all", exact=True).click()
+    expect(dialog.locator(".modal-meta")).to_have_text("3 of 3 selected")
+    # Move title after score using the same pointer events as mouse/touch dragging.
+    handle = dialog.locator(".drag-handle").first.bounding_box()
+    target = dialog.locator(".drag-item").last.bounding_box()
+    page.mouse.move(handle["x"] + handle["width"] / 2, handle["y"] + 24)
+    page.mouse.down()
+    page.mouse.move(target["x"] + 24, target["y"] + target["height"] - 4, steps=5)
+    expect(dialog.locator(".drag-ghost")).to_be_visible()
+    page.mouse.up()
+    expect(dialog.locator(".drag-item-label")).to_have_text(["notes", "score", "title"])
+    dialog.get_by_role("button", name="Apply", exact=True).click()
+    expect(dialog).not_to_be_visible()
+    assert page.evaluate("appliedColumns") == ["notes", "score", "title"]
+    chooser.evaluate(
+        "node => node.open({columns: ['title', 'notes'], selected: ['title']})"
+    )
+    dialog.get_by_role("button", name="Deselect all", exact=True).click()
+    dialog.get_by_role("button", name="Cancel", exact=True).click()
+    expect(dialog).not_to_be_visible()
+    assert page.evaluate("appliedColumns") == ["notes", "score", "title"]
+
+
 @pytest.mark.playwright
 def test_insert_row_flow_uses_custom_column_field(page, datasette_server):
     page.add_init_script("""
@@ -1599,8 +1689,615 @@ def test_delete_row_flow_removes_row(page, datasette_server):
     dialog = page.locator("#row-delete-dialog")
     dialog.wait_for()
     assert "Delete row 1" in dialog.inner_text()
-    dialog.locator(".row-delete-confirm").click()
+    dialog.locator(".row-delete-confirm").press("Enter")
 
     page.locator(".row-mutation-status", has_text="Deleted row 1").wait_for()
     page.locator('tr[data-row="1"]').wait_for(state="detached")
     assert project_rows(datasette_server, id=1) == []
+
+
+@pytest.mark.playwright
+def test_count_all(page, datasette_server):
+    page.goto(datasette_server + "data/count_numbers?id__gt=1&_sort=id")
+    assert page.locator(".table-count").inner_text() == "10,000+ rows"
+    with page.expect_response("**/count_numbers/-/count?*") as response:
+        page.get_by_role("button", name="count all", exact=True).click()
+    assert response.value.request.method == "POST"
+    assert response.value.request.post_data is None
+    assert "content-type" not in response.value.request.headers
+    assert response.value.json() == {"ok": True, "count": 10001}
+    page.wait_for_function(
+        'document.querySelector(".table-count").textContent === "10,001 rows"'
+    )
+    assert page.locator(".count-all").count() == 0
+    assert "id" in page.locator("h3").first.inner_text()
+
+
+@pytest.mark.playwright
+def test_count_all_error_retry(page, datasette_server):
+    page.goto(datasette_server + "data/count_numbers?id__gt=1")
+    page.route(
+        "**/count_numbers/-/count?*",
+        lambda route: route.fulfill(
+            status=400,
+            content_type="application/json",
+            body=json.dumps({"ok": False, "errors": ["Count query timed out"]}),
+        ),
+    )
+    button = page.get_by_role("button", name="count all", exact=True)
+    button.click()
+    page.wait_for_function(
+        'document.querySelector(".count-error").textContent === "Count query timed out"'
+    )
+    assert button.is_enabled()
+    page.unroute("**/count_numbers/-/count?*")
+    button.click()
+    page.wait_for_function(
+        'document.querySelector(".table-count").textContent === "10,001 rows"'
+    )
+    assert page.locator(".count-error").inner_text() == ""
+
+
+@pytest.mark.playwright
+def test_modal_lifecycle(page, datasette_server):
+    from playwright.sync_api import expect
+
+    page.goto(datasette_server)
+    page.evaluate(
+        """() => {
+            const trigger = document.createElement('button');
+            trigger.id = 'modal-trigger';
+            trigger.textContent = 'Open test modal';
+            document.body.append(trigger);
+            window.testModal = DatasetteModal.create();
+            const dialog = testModal.dialog;
+            dialog.id = 'test-modal';
+            dialog.setAttribute('aria-labelledby', 'test-modal-title');
+            dialog.innerHTML = `
+                <h2 id="test-modal-title">Test modal</h2>
+                <input aria-label="First field">
+                <input aria-label="Second field">
+                <button id="test-modal-cancel">Cancel</button>`;
+            // Padding is part of the dialog, never a backdrop dismissal.
+            dialog.style.padding = '30px';
+            document.body.append(testModal);
+            window.closeSources = [];
+            testModal.beforeClose = source => {
+                closeSources.push(source);
+                return window.allowClose;
+            };
+            window.allowClose = false;
+            trigger.onclick = () => testModal.show({
+                returnFocusTo: trigger, initialFocus: dialog.querySelector('input')
+            });
+            dialog.querySelector('button').onclick = () => testModal.requestClose('cancel');
+        }""",
+    )
+    trigger = page.locator("#modal-trigger")
+    trigger.click()
+    dialog = page.get_by_role("dialog", name="Test modal", exact=True)
+    expect(dialog.get_by_role("textbox", name="First field")).to_be_focused()
+    assert dialog.evaluate("node => node instanceof HTMLDialogElement")
+    expect(dialog).to_have_css("display", "flex")
+    # Native modality keeps background content inert and keyboard focus inside.
+    page.keyboard.press("Tab")
+    expect(dialog.get_by_role("textbox", name="Second field")).to_be_focused()
+    page.keyboard.press("Shift+Tab")
+    expect(dialog.get_by_role("textbox", name="First field")).to_be_focused()
+    trigger.evaluate("node => node.focus()")
+    expect(dialog.get_by_role("textbox", name="First field")).to_be_focused()
+
+    page.keyboard.down("Escape")
+    assert page.evaluate("closeSources") == []
+    page.keyboard.up("Escape")
+    page.wait_for_function("closeSources.length === 1")
+    assert page.evaluate("closeSources") == ["escape"]
+    expect(dialog).to_be_visible()
+
+    dialog.click(position={"x": 3, "y": 3})
+    assert page.evaluate("closeSources") == ["escape"]
+    # A drag which starts inside and ends on the backdrop must not dismiss.
+    box = dialog.bounding_box()
+    page.mouse.move(box["x"] + 3, box["y"] + 3)
+    page.mouse.down()
+    page.mouse.move(2, 2)
+    page.mouse.up()
+    assert page.evaluate("closeSources") == ["escape"]
+    page.mouse.click(2, 2)
+    assert page.evaluate("closeSources") == ["escape", "backdrop"]
+
+    page.evaluate("testModal.busy = true; allowClose = true")
+    expect(dialog).to_have_attribute("aria-busy", "true")
+    page.keyboard.press("Escape")
+    page.mouse.click(2, 2)
+    dialog.get_by_role("button", name="Cancel").click()
+    expect(dialog).to_be_visible()
+    assert page.evaluate("closeSources") == ["escape", "backdrop"]
+    page.evaluate("testModal.busy = false")
+    dialog.get_by_role("button", name="Cancel").click()
+    expect(dialog).not_to_be_visible()
+    expect(trigger).to_be_focused()
+    assert page.evaluate("closeSources") == ["escape", "backdrop", "cancel"]
+
+    # Reopening, including an extra show() call, preserves the original return-focus target.
+    trigger.click()
+    page.evaluate("testModal.show()")
+    page.keyboard.press("Escape")
+    expect(dialog).not_to_be_visible()
+    expect(trigger).to_be_focused()
+
+    # Completion bypasses busy/confirmation and must not steal a caller's focus.
+    trigger.click()
+    page.evaluate("""() => new Promise(resolve => {
+        const next = document.createElement('button');
+        next.id = 'after-save';
+        next.textContent = 'Next action';
+        document.body.append(next);
+        testModal.dialog.addEventListener('close', resolve, {once: true});
+        testModal.busy = true;
+        testModal.close({restoreFocus: false});
+        next.focus();
+    })""")
+    expect(dialog).not_to_be_visible()
+    expect(page.locator("#after-save")).to_be_focused()
+
+
+@pytest.mark.playwright
+def test_modal_nested_escape_and_cleanup(page, datasette_server):
+    from playwright.sync_api import expect
+
+    page.goto(datasette_server + "data/projects")
+    trigger = page.locator('tr[data-row="1"] button[data-row-action="edit"]')
+    trigger.click()
+    dialog = page.locator("#row-edit-dialog")
+    field = dialog.locator('input[name="title"]')
+    expect(field).to_be_visible()
+    field.fill("Unsaved title")
+    page.evaluate("""() => {
+        window.confirmations = [];
+        window.confirm = message => { confirmations.push(message); return false; };
+    }""")
+    # Plugin controls can consume Escape without closing their containing form.
+    field.evaluate("""node => node.addEventListener('keydown', event => {
+        if (event.key === 'Escape') event.preventDefault();
+    }, {once: true})""")
+    field.press("Escape")
+    assert page.evaluate("confirmations") == []
+    expect(dialog).to_be_visible()
+    field.press("Escape")
+    page.wait_for_function("confirmations.length === 1")
+    assert page.evaluate("confirmations") == ["Discard unsaved changes to this row?"]
+
+    # A nested native modal closes independently, then returns focus to its field.
+    field.evaluate("""node => {
+        node.focus();
+        window.nestedModal = DatasetteModal.create();
+        nestedModal.dialog.setAttribute('aria-label', 'Nested picker');
+        nestedModal.dialog.innerHTML = '<button>Choose</button>';
+        node.closest('dialog').append(nestedModal);
+        nestedModal.show();
+    }""")
+    nested = page.get_by_role("dialog", name="Nested picker")
+    page.keyboard.press("Escape")
+    expect(nested).not_to_be_visible()
+    expect(dialog).to_be_visible()
+    expect(field).to_be_focused()
+    assert page.evaluate("confirmations.length") == 1
+
+    # Closing before keyup cancels the pending confirmation, including on reopen.
+    page.keyboard.down("Escape")
+    dialog.locator(".row-edit-cancel").click()
+    expect(dialog).not_to_be_visible()
+    trigger.click()
+    page.keyboard.up("Escape")
+    expect(field).to_be_visible()
+    assert page.evaluate("confirmations.length") == 1
+    expect(dialog).to_be_visible()
+    # Native cancel (e.g. an accessibility action) does not wait for keyboard input.
+    field.fill("Another edit")
+    dialog.evaluate(
+        "node => node.dispatchEvent(new Event('cancel', {cancelable: true}))"
+    )
+    assert page.evaluate("confirmations.length") == 2
+    dialog.locator(".row-edit-cancel").click()
+    expect(trigger).to_be_focused()
+
+
+@pytest.mark.playwright
+@pytest.mark.parametrize("name", ["jump", "columns", "type", "mobile"])
+def test_modal_consumers_dismiss_and_restore_focus(page, datasette_server, name):
+    from playwright.sync_api import expect
+
+    page_errors = []
+    page.on("pageerror", lambda error: page_errors.append(str(error)))
+    if name == "mobile":
+        page.set_viewport_size({"width": 390, "height": 844})
+    page.emulate_media(reduced_motion="reduce")
+    page.goto(datasette_server + "data/projects")
+    if name == "jump":
+        trigger = page.locator("details.nav-menu summary")
+        trigger.click()
+        page.locator("[data-navigation-search-open]").click()
+        dialog = page.locator("navigation-search dialog")
+    elif name == "columns":
+        # Open through its public API with a real, focused page control.
+        trigger = page.locator("details.actions-menu-links summary")
+        trigger.focus()
+        page.evaluate(
+            "document.querySelector('column-chooser').open({columns: ['id', 'title'], selected: ['id']})"
+        )
+        dialog = page.locator("column-chooser dialog")
+    elif name == "type":
+        trigger = page.locator("details.actions-menu-links summary")
+        trigger.focus()
+        page.evaluate(
+            "openSetColumnTypeDialog(document.querySelector('th[data-column=title]'))"
+        )
+        dialog = page.locator("#set-column-type-dialog")
+    else:
+        trigger = page.locator(".column-actions-mobile")
+        trigger.click()
+        dialog = page.locator("#mobile-column-actions-dialog")
+    expect(dialog).to_be_visible()
+    expect(dialog).to_have_css("border-radius", "8px" if name == "mobile" else "12px")
+    expect(dialog).to_have_css("animation-name", "none")
+    assert dialog.evaluate("node => node.parentElement.localName") == "datasette-modal"
+    assert dialog.evaluate("node => node.getRootNode() === document")
+    page.keyboard.press("Escape")
+    expect(dialog).not_to_be_visible()
+    expect(trigger).to_be_focused()
+    assert page_errors == []
+
+
+@pytest.mark.playwright
+def test_modal_disconnect_cleans_up_pending_escape(page, datasette_server):
+    from playwright.sync_api import expect
+
+    page.goto(datasette_server)
+    page.evaluate("""() => {
+        window.detachable = DatasetteModal.create();
+        detachable.dialog.setAttribute('aria-label', 'Detachable');
+        detachable.dialog.innerHTML = '<button>Focus</button>';
+        window.closeAttempts = 0;
+        detachable.beforeClose = () => { closeAttempts++; return false; };
+        document.body.append(detachable);
+        detachable.show();
+    }""")
+    dialog = page.get_by_role("dialog", name="Detachable")
+    page.keyboard.down("Escape")
+    page.evaluate("detachable.remove()")
+    page.keyboard.up("Escape")
+    assert page.evaluate("closeAttempts") == 0
+    assert page.evaluate("detachable.dialog.open") is False
+    page.evaluate("document.body.append(detachable); detachable.show()")
+    expect(dialog).to_be_visible()
+    page.keyboard.press("Escape")
+    page.wait_for_function("closeAttempts === 1")
+    expect(dialog).to_be_visible()
+
+
+@pytest.mark.playwright
+@pytest.mark.parametrize("kind", ["create", "alter"])
+def test_schema_modal_escape_confirmation_and_focus(page, datasette_server, kind):
+    from playwright.sync_api import expect
+
+    path = "data" if kind == "create" else "data/projects"
+    page.goto(datasette_server + path)
+    menu = page.locator("details.actions-menu-links")
+    menu.locator("summary").click()
+    selector = "data-database-action" if kind == "create" else "data-table-action"
+    menu.locator(f'button[{selector}="{kind}-table"]').click()
+    dialog = page.locator(f"#table-{kind}-dialog")
+    if kind == "create":
+        dialog.locator('input[name="table"]').fill("unsaved_table")
+    else:
+        dialog.locator(".table-alter-add-column").click()
+    # Real browser confirms, including WebKit, should appear once and stay usable.
+    confirmations = []
+
+    def reject(prompt):
+        confirmations.append(prompt.message)
+        prompt.dismiss()
+
+    page.on("dialog", reject)
+    with page.expect_event("dialog"):
+        page.keyboard.press("Escape")
+    expect(dialog).to_be_visible()
+    assert len(confirmations) == 1
+    page.remove_listener("dialog", reject)
+    page.on("dialog", lambda prompt: prompt.accept())
+    page.keyboard.press("Escape")
+    expect(dialog).not_to_be_visible()
+    expect(menu.locator("summary")).to_be_focused()
+
+
+MENU_CASES = [
+    ("app", "", "details.nav-menu summary", "#app-menu-panel"),
+    ("database", "data", "details.actions-menu-links summary", "#actions-menu-panel"),
+    (
+        "column",
+        "data/projects",
+        'th[data-column="title"] .column-menu-trigger',
+        "#column-actions-menu",
+    ),
+]
+
+
+@pytest.mark.playwright
+@pytest.mark.parametrize("width", [390, 1280])
+def test_menu_patterns_use_shared_menus(page, datasette_server, width):
+    from playwright.sync_api import expect
+
+    page.set_viewport_size({"width": width, "height": 1000})
+    page.goto(datasette_server + "-/patterns/menus")
+    samples = page.locator('.menu-pattern-open [role="menu"]')
+    expect(samples).to_have_count(3)
+    for sample in samples.all():
+        expect(sample).to_be_visible()
+    for kind in ("app", "database", "column"):
+        panel_id = f"closed-{kind}-panel"
+        trigger = page.locator(f'[aria-controls="{panel_id}"]')
+        trigger.click()
+        panel = page.locator(f"#{panel_id}")
+        expect(panel).to_be_visible()
+        assert panel.evaluate(
+            "node => node.classList.contains('datasette-menu-floating')"
+        )
+        expect(trigger).to_be_focused()
+        page.keyboard.press("ArrowDown")
+        expect(panel.get_by_role("menuitem").first).to_be_focused()
+        panel.get_by_role("menuitem").first.click()
+        expect(panel).not_to_be_visible()
+        expect(page).to_have_url(datasette_server + "-/patterns/menus")
+    trigger = page.locator(".menu-pattern-closed .column-menu-trigger")
+    assert trigger.bounding_box()["height"] == 24
+    expect(page.locator(".menu-pattern-closed details[open]")).to_have_count(0)
+
+
+@pytest.mark.playwright
+@pytest.mark.parametrize("width", [390, 1280])
+def test_portfolio_uses_shared_menus(page, datasette_server, width):
+    from playwright.sync_api import expect
+
+    page.set_viewport_size({"width": width, "height": 1000})
+    page.goto(datasette_server + "-/patterns")
+    for panel_id in (
+        "app-menu-panel",
+        "pattern-database-actions",
+        "pattern-table-actions",
+    ):
+        trigger = page.locator(f'[aria-controls="{panel_id}"]')
+        trigger.click()
+        panel = page.locator(f"#{panel_id}")
+        expect(panel).to_be_visible()
+        assert panel.evaluate(
+            "node => node.classList.contains('datasette-menu-floating')"
+        )
+        expect(trigger).to_be_focused()
+        page.keyboard.press("ArrowDown")
+        expect(panel.get_by_role("menuitem").first).to_be_focused()
+        page.keyboard.press("Escape")
+        expect(panel).not_to_be_visible()
+        expect(trigger).to_be_focused()
+
+
+@pytest.mark.playwright
+@pytest.mark.parametrize("width", [390, 590])
+@pytest.mark.parametrize(
+    "path,trigger_selector,panel_selector",
+    [
+        ("", "details.nav-menu summary", "#app-menu-panel"),
+        ("data", "details.actions-menu-links summary", "#actions-menu-panel"),
+        ("data/projects", "details.actions-menu-links summary", "#actions-menu-panel"),
+    ],
+)
+def test_narrow_menus_stay_near_trigger(
+    page, datasette_server, width, path, trigger_selector, panel_selector
+):
+    from playwright.sync_api import expect
+
+    page.set_viewport_size({"width": width, "height": 1000})
+    page.goto(datasette_server + path)
+    trigger = page.locator(trigger_selector)
+    trigger.click()
+    panel = page.locator(panel_selector)
+    expect(panel).to_be_visible()
+    anchor = trigger.bounding_box()
+    menu = panel.bounding_box()
+    gap = 0 if panel_selector == "#app-menu-panel" else 8
+    assert menu["y"] - (anchor["y"] + anchor["height"]) == pytest.approx(gap, abs=1)
+    assert menu["x"] >= 12
+    assert menu["x"] + menu["width"] <= width - 12
+
+
+@pytest.mark.playwright
+@pytest.mark.parametrize("kind,path,trigger_selector,panel_selector", MENU_CASES)
+@pytest.mark.parametrize("width,height", [(1280, 800), (590, 500), (320, 240)])
+def test_menus_fit_viewport(
+    page, datasette_server, kind, path, trigger_selector, panel_selector, width, height
+):
+    from playwright.sync_api import expect
+
+    errors = []
+    page.on("pageerror", lambda error: errors.append(str(error)))
+    # Open before resizing: at phone widths the table uses cards and a separate
+    # column picker, while an already-open column menu should still fit.
+    page.set_viewport_size({"width": 1280, "height": 800})
+    page.goto(datasette_server + path)
+    trigger = page.locator(trigger_selector)
+    trigger.click()
+    panel = page.locator(panel_selector)
+    expect(panel).to_be_visible()
+    page.set_viewport_size({"width": width, "height": height})
+    page.wait_for_function(
+        """selector => {
+        const r = document.querySelector(selector).getBoundingClientRect();
+        return r.left >= 11 && r.top >= 11 &&
+            r.right <= innerWidth - 11 && r.bottom <= innerHeight - 11;
+    }""",
+        arg=panel_selector,
+    )
+    assert panel.evaluate("node => node.scrollWidth <= node.clientWidth + 1")
+    if kind == "column" and height == 240:
+        assert panel.evaluate("node => node.scrollHeight > node.clientHeight")
+        page.keyboard.press("End")
+        assert panel.evaluate("node => node.contains(document.activeElement)")
+        assert panel.evaluate("node => node.scrollTop > 0")
+    if kind != "column":
+        # These controls are available directly on phones, too.
+        page.keyboard.press("Escape")
+        trigger.click()
+        expect(panel).to_be_visible()
+        if width <= 600:
+            minimum_height = 44 if kind == "app" else 48
+            assert panel.locator("[role=menuitem]").first.evaluate(
+                "(node, minimum) => node.getBoundingClientRect().height >= minimum",
+                minimum_height,
+            )
+    page.keyboard.press("Escape")
+    expect(panel).not_to_be_visible()
+    expect(trigger).to_have_attribute("aria-expanded", "false")
+    assert errors == []
+
+
+@pytest.mark.playwright
+@pytest.mark.parametrize("kind,path,trigger_selector,panel_selector", MENU_CASES)
+@pytest.mark.parametrize("key", ["ArrowDown", "ArrowUp"])
+def test_menu_pointer_open_waits_for_keyboard_navigation(
+    page, datasette_server, kind, path, trigger_selector, panel_selector, key
+):
+    from playwright.sync_api import expect
+
+    page.goto(datasette_server + path)
+    trigger = page.locator(trigger_selector)
+    panel = page.locator(panel_selector)
+    trigger.click()
+    expect(panel).to_be_visible()
+    expect(trigger).to_be_focused()
+    assert not panel.evaluate("node => node.contains(document.activeElement)")
+    page.keyboard.press(key)
+    items = panel.locator("[role=menuitem]")
+    expect(items.first if key == "ArrowDown" else items.last).to_be_focused()
+    page.keyboard.press("Escape")
+    expect(panel).not_to_be_visible()
+    expect(trigger).to_be_focused()
+
+
+@pytest.mark.playwright
+@pytest.mark.parametrize("kind,path,trigger_selector,panel_selector", MENU_CASES)
+def test_menu_keyboard_and_dismissal(
+    page, datasette_server, kind, path, trigger_selector, panel_selector
+):
+    from playwright.sync_api import expect
+
+    page.goto(datasette_server + path)
+    trigger = page.locator(trigger_selector)
+    panel = page.locator(panel_selector)
+    trigger.focus()
+    page.keyboard.press("ArrowDown")
+    expect(panel).to_be_visible()
+    items = panel.locator("[role=menuitem]")
+    expect(items.first).to_be_focused()
+    page.keyboard.press("End")
+    expect(items.last).to_be_focused()
+    page.keyboard.press("ArrowDown")
+    expect(items.first).to_be_focused()
+    page.keyboard.press("Escape")
+    expect(panel).not_to_be_visible()
+    expect(trigger).to_be_focused()
+    page.keyboard.press("ArrowUp")
+    expect(items.last).to_be_focused()
+    trigger.click()
+    expect(panel).not_to_be_visible()
+    expect(trigger).to_be_focused()
+    trigger.click()
+    page.locator("h1").click()
+    expect(panel).not_to_be_visible()
+    trigger.click()
+    page.keyboard.press("Tab")
+    expect(panel).not_to_be_visible()
+
+
+@pytest.mark.playwright
+def test_column_menu_actions_and_dialog_focus(page, datasette_server):
+    from playwright.sync_api import expect
+
+    page.goto(datasette_server + "data/projects")
+    trigger = page.locator('th[data-column="title"] .column-menu-trigger')
+    trigger.click()
+    panel = page.locator("#column-actions-menu")
+    panel.get_by_role("menuitem", name="Choose columns", exact=True).click()
+    dialog = page.locator("column-chooser dialog")
+    expect(dialog).to_be_visible()
+    expect(panel).not_to_be_visible()
+    page.keyboard.press("Escape")
+    expect(dialog).not_to_be_visible()
+    expect(trigger).to_be_focused()
+    trigger.click()
+    panel.get_by_role("menuitem", name="Sort descending", exact=True).click()
+    expect(page).to_have_url(datasette_server + "data/projects?_sort_desc=title")
+    trigger.click()
+    panel.get_by_role("menuitem", name="Hide this column", exact=True).click()
+    expect(page.locator('th[data-column="title"]')).to_have_count(0)
+
+
+@pytest.mark.playwright
+def test_menu_switching_and_header_alignment(page, datasette_server):
+    from playwright.sync_api import expect
+
+    page.goto(datasette_server + "data/projects")
+    # The signed-in label uses the same header layout as the anonymous fixture.
+    page.locator("header.app-header nav").evaluate("""node => {
+        const actor = document.createElement('div');
+        actor.className = 'actor';
+        actor.innerHTML = '<strong>root</strong>';
+        node.append(actor);
+    }""")
+    assert page.locator("header.app-header").evaluate("""node => {
+        const actor = node.querySelector('.actor').getBoundingClientRect();
+        const icon = node.querySelector('summary svg').getBoundingClientRect();
+        return Math.abs(actor.top + actor.height / 2 - icon.top - icon.height / 2) < 1;
+    }""")
+    app = page.locator("details.nav-menu summary")
+    actions = page.locator("details.actions-menu-links summary")
+    app.click()
+    actions.click()
+    expect(page.locator("#app-menu-panel")).not_to_be_visible()
+    expect(page.locator("#actions-menu-panel")).to_be_visible()
+    assert page.locator(".datasette-menu-floating").count() == 1
+    page.keyboard.press("Escape")
+    assert page.locator("header.app-header").evaluate("""node =>
+        Math.abs(node.getBoundingClientRect().height -
+            2.6 * parseFloat(getComputedStyle(document.documentElement).fontSize)) < 1
+    """)
+
+
+@pytest.mark.playwright
+def test_menu_long_plugin_content_and_no_popover_fallback(page, datasette_server):
+    from playwright.sync_api import expect
+
+    page.add_init_script("HTMLElement.prototype.showPopover = undefined")
+    page.set_viewport_size({"width": 390, "height": 360})
+    page.goto(datasette_server)
+    # Emulate plugin links with long, unbroken names and enough items to scroll.
+    page.locator("#app-menu-panel ul").evaluate("""node => {
+        for (let i = 0; i < 20; i++) {
+            const li = document.createElement('li');
+            const a = document.createElement('a');
+            a.href = '#plugin';
+            a.role = 'menuitem';
+            a.textContent = 'Plugin-' + i + '-' + 'x'.repeat(100);
+            li.append(a);
+            node.append(li);
+        }
+    }""")
+    page.locator("details.nav-menu summary").click()
+    panel = page.locator("#app-menu-panel")
+    expect(panel).to_be_visible()
+    assert panel.evaluate("node => node.scrollWidth <= node.clientWidth + 1")
+    assert panel.evaluate("node => node.scrollHeight > node.clientHeight")
+    page.keyboard.press("End")
+    expect(panel.get_by_role("menuitem").last).to_be_focused()
+    assert panel.evaluate("node => node.scrollTop > 0")
+    page.keyboard.press("Escape")
+    expect(panel).not_to_be_visible()

tests/test_plugins.py

@@ -1,21 +1,3 @@
-from bs4 import BeautifulSoup as Soup
-from .fixtures import (
-    make_app_client,
-    TEMP_PLUGIN_SECRET_FILE,
-    PLUGINS_DIR,
-    TestClient as _TestClient,
-)  # noqa
-from click.testing import CliRunner
-from datasette.app import Datasette
-from datasette import cli, hookimpl
-from datasette.fixtures import TABLES
-from datasette.filters import FilterArguments
-from datasette.plugins import get_plugins, DEFAULT_PLUGINS, pm
-from datasette.permissions import PermissionSQL, Action
-from datasette.resources import DatabaseResource
-from datasette.utils.sqlite import sqlite3
-from datasette.utils import StartupError, await_me_maybe
-from jinja2 import ChoiceLoader, FileSystemLoader
 import base64
 import datetime
 import importlib
@@ -24,9 +6,32 @@ import os
 import pathlib
 import re
 import textwrap
-import pytest
 import urllib
 
+import pytest
+from bs4 import BeautifulSoup as Soup
+from click.testing import CliRunner
+from jinja2 import ChoiceLoader, FileSystemLoader
+
+from datasette import cli, hookimpl
+from datasette.app import Datasette
+from datasette.filters import FilterArguments
+from datasette.fixtures import TABLES
+from datasette.permissions import Action, PermissionSQL
+from datasette.plugins import DEFAULT_PLUGINS, get_plugins, pm
+from datasette.resources import DatabaseResource
+from datasette.utils import StartupError, await_me_maybe
+from datasette.utils.sqlite import sqlite3
+
+from .fixtures import (
+    PLUGINS_DIR,
+    TEMP_PLUGIN_SECRET_FILE,
+    make_app_client,
+)
+from .fixtures import (
+    TestClient as _TestClient,
+)
+
 at_memory_re = re.compile(r" at 0x\w+")
 
 
@@ -35,7 +40,7 @@ at_memory_re = re.compile(r" at 0x\w+")
 )
 def test_plugin_hooks_have_tests(plugin_hook):
     """Every plugin hook should be referenced in this test module"""
-    tests_in_this_module = [t for t in globals().keys() if t.startswith("test_hook_")]
+    tests_in_this_module = [t for t in globals() if t.startswith("test_hook_")]
     ok = False
     for test in tests_in_this_module:
         if plugin_hook in test:
@@ -48,6 +53,13 @@ def test_hook_jump_items_sql():
     assert "jump_items_sql" in dir(pm.hook)
 
 
+def test_hook_shutdown():
+    # Detailed behavior (ordering against background-task cancellation and
+    # close(), idempotency, exception handling, sync vs async support) is
+    # covered in tests/test_shutdown.py.
+    assert "shutdown" in dir(pm.hook)
+
+
 @pytest.mark.asyncio
 async def test_hook_plugins_dir_plugin_prepare_connection(ds_client):
     response = await ds_client.get(
@@ -125,11 +137,11 @@ async def test_hook_extra_css_urls(ds_client, path, expected_decoded_object):
     response = await ds_client.get(path)
     assert response.status_code == 200
     links = Soup(response.text, "html.parser").find_all("link")
-    special_href = [
+    special_href = next(
         link
         for link in links
         if link.attrs["href"].endswith("/extra-css-urls-demo.css")
-    ][0]["href"]
+    )["href"]
     # This link has a base64-encoded JSON blob in it
     encoded = special_href.split("/")[3]
     actual_decoded_object = json.loads(base64.b64decode(encoded).decode("utf8"))
@@ -152,7 +164,7 @@ async def test_hook_extra_js_urls(ds_client):
             "type": "module",
         },
     ]:
-        assert any(s == attrs for s in script_attrs), "Expected: {}".format(attrs)
+        assert any(s == attrs for s in script_attrs), f"Expected: {attrs}"
 
 
 @pytest.mark.asyncio
@@ -315,7 +327,8 @@ async def test_plugin_config_env_from_list(ds_client):
 
 @pytest.mark.asyncio
 async def test_plugin_config_file(ds_client):
-    with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp:
+    # Blocking write is fine here - it is tiny test setup, not request handling
+    with open(TEMP_PLUGIN_SECRET_FILE, "w") as fp:  # noqa: ASYNC230
         fp.write("FROM_FILE")
     assert {"foo": "FROM_FILE"} == ds_client.ds.plugin_config("file-plugin")
     os.remove(TEMP_PLUGIN_SECRET_FILE)
@@ -416,6 +429,72 @@ def test_hook_extra_template_vars(restore_working_directory):
         } == extra_template_vars_from_awaitable
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "return_style", ["direct", "callable", "async_callable", "awaitable"]
+)
+async def test_hook_extra_template_vars_none(ds_client, return_style):
+    class OtherPlugin:
+        @hookimpl
+        def extra_template_vars(self):
+            return {"other": "present"}
+
+    class ConditionalPlugin:
+        @hookimpl
+        def extra_template_vars(self, view_name):
+            def inner():
+                if view_name == "database":
+                    return {"conditional": "database"}
+
+            async def async_inner():
+                return inner()
+
+            if return_style == "direct":
+                return inner()
+            elif return_style == "callable":
+                return inner
+            elif return_style == "async_callable":
+                return async_inner
+            else:
+                return async_inner()
+
+    other_plugin = OtherPlugin()
+    conditional_plugin = ConditionalPlugin()
+    pm.register(other_plugin)
+    pm.register(conditional_plugin)
+    try:
+        template = ds_client.ds.get_jinja_environment().from_string(
+            "{{ other }}:{{ conditional|default('missing') }}"
+        )
+        for view_name, expected in (
+            ("database", "present:database"),
+            ("index", "present:missing"),
+        ):
+            rendered = await ds_client.ds.render_template(template, view_name=view_name)
+            assert rendered == expected
+    finally:
+        pm.unregister(conditional_plugin)
+        pm.unregister(other_plugin)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("invalid_value", [False, 0, "", [], ()])
+async def test_hook_extra_template_vars_invalid(ds_client, invalid_value):
+    class InvalidPlugin:
+        @hookimpl
+        def extra_template_vars(self):
+            return lambda: invalid_value
+
+    plugin = InvalidPlugin()
+    pm.register(plugin)
+    try:
+        template = ds_client.ds.get_jinja_environment().from_string("test")
+        with pytest.raises(AssertionError, match="extra_vars is of type"):
+            await ds_client.ds.render_template(template)
+    finally:
+        pm.unregister(plugin)
+
+
 def test_plugins_async_template_function(restore_working_directory):
     with make_app_client(
         template_dir=str(pathlib.Path(__file__).parent / "test_templates")
@@ -823,7 +902,7 @@ def test_hook_register_routes_with_datasette(configured_path):
         assert response.status_code == 200
         assert configured_path.upper() == response.text
         # Other one should 404
-        other_path = [p for p in ("path1", "path2") if configured_path != p][0]
+        other_path = next(p for p in ("path1", "path2") if configured_path != p)
         assert client.get(f"/{other_path}/", follow_redirects=True).status_code == 404
 
 
@@ -928,7 +1007,7 @@ async def test_plugin_startup_can_add_queries():
                 await datasette.add_query(
                     "data",
                     "from_startup",
-                    "select {}".format(result.first()[0]),
+                    f"select {result.first()[0]}",
                     source="plugin",
                 )
 
@@ -1040,7 +1119,7 @@ async def test_hook_handle_exception(ds_client):
 @pytest.mark.asyncio
 @pytest.mark.parametrize("param", ("_custom_error", "_custom_error_async"))
 async def test_hook_handle_exception_custom_response(ds_client, param):
-    response = await ds_client.get("/trigger-error?{}=1".format(param))
+    response = await ds_client.get(f"/trigger-error?{param}=1")
     assert response.text == param
 
 
@@ -1294,7 +1373,8 @@ async def test_hook_filters_from_request(ds_client):
 
     ds_client.ds.pm.register(ReturnNothingPlugin(), name="ReturnNothingPlugin")
     response = await ds_client.get("/fixtures/facetable?_nothing=1")
-    assert "0 rows\n        where NOTHING" in response.text
+    summary = Soup(response.text, "html.parser").select_one(".table-summary")
+    assert summary.get_text(" ", strip=True) == "0 rows where NOTHING"
     json_response = await ds_client.get("/fixtures/facetable.json?_nothing=1")
     assert json_response.json()["rows"] == []
     ds_client.ds.pm.unregister(name="ReturnNothingPlugin")
@@ -1374,7 +1454,7 @@ async def test_hook_register_actions_no_duplicates(duplicate):
     # This should error:
     with pytest.raises(StartupError) as ex:
         await ds.invoke_startup()
-        assert "Duplicate action {}".format(duplicate) in str(ex.value)
+        assert f"Duplicate action {duplicate}" in str(ex.value)
 
 
 @pytest.mark.asyncio
@@ -1482,7 +1562,7 @@ async def test_plugin_is_installed():
         datasette.pm.register(DummyPlugin(), name="DummyPlugin")
         response = await datasette.client.get("/-/plugins.json")
         assert response.status_code == 200
-        installed_plugins = {p["name"] for p in response.json()["plugins"]}
+        installed_plugins = {p["name"] for p in response.json()}
         assert "DummyPlugin" in installed_plugins
 
     finally:

tests/test_pr76_fts_policy.py

@@ -0,0 +1,140 @@
+"""Policy and compatibility coverage for PR #76, run against the fixed checkout."""
+
+import uuid
+
+import pytest
+
+from datasette.app import Datasette
+from datasette.resources import TableResource
+from datasette.utils.sqlite import sqlite3, sqlite_derived_table_dependencies
+
+
+@pytest.mark.parametrize("vocab_name", ["words", "name USING fts4aux", 'quoted"name'])
+@pytest.mark.parametrize(
+    "module,arguments",
+    [
+        ("fts5vocab", "'Search,Index', 'row'"),
+        ("fts5vocab", "'SEARCH,INDEX', 'col'"),
+        ("fts5vocab", "'Search,Index', 'instance'"),
+        ("fts4aux", "'Search,Index'"),
+    ],
+)
+def test_vocabulary_dependency_identity(module, arguments, vocab_name):
+    conn = sqlite3.connect(":memory:")
+    try:
+        fts = "fts5" if module == "fts5vocab" else "fts4"
+        conn.execute(f'create virtual table "Search,Index" using {fts}(body)')
+        quoted_name = '"' + vocab_name.replace('"', '""') + '"'
+        conn.execute(
+            f"create virtual table {quoted_name} USING /* module */ {module}({arguments})"
+        )
+        assert sqlite_derived_table_dependencies(conn)[vocab_name] == "Search,Index"
+    finally:
+        conn.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("module", ["fts5", "fts4"])
+@pytest.mark.parametrize("external_content", [False, True], ids=["one-hop", "two-hop"])
+@pytest.mark.parametrize(
+    "source_allowed,vocab_allowed", [(False, True), (True, False), (True, True)]
+)
+async def test_vocabulary_immediate_source_permissions(
+    module, external_content, source_allowed, vocab_allowed
+):
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "search": {
+                            "permissions": {
+                                "view-table": (
+                                    {"id": "reader"} if source_allowed else False
+                                )
+                            }
+                        },
+                        "words": {"permissions": {"view-table": vocab_allowed}},
+                    }
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.execute_write("create table documents(body text)")
+    options = "body, content='documents'" if external_content else "body"
+    await db.execute_write(f"create virtual table search using {module}({options})")
+    definition = (
+        "fts5vocab('SEARCH', 'row')" if module == "fts5" else "fts4aux('SEARCH')"
+    )
+    await db.execute_write(f"create virtual table words using {definition}")
+    await ds.invoke_startup()
+    try:
+        actor = {"id": "reader"}
+        expected = source_allowed and vocab_allowed and not external_content
+        for name in ("words", "WORDS"):
+            assert (
+                await ds.allowed(
+                    action="view-table",
+                    resource=TableResource("data", name),
+                    actor=actor,
+                )
+                is expected
+            )
+        resources = await ds.allowed_resources(
+            "view-table", parent="data", actor=actor, include_is_private=True
+        )
+        words = [r for r in resources.resources if r.child == "words"]
+        assert bool(words) is expected
+        if expected:
+            assert words[0].private
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "words")
+        )
+        # Dropping the source invalidates dependency metadata and remains denied.
+        await db.execute_write("drop table search")
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "words"), actor=actor
+        )
+    finally:
+        ds.close()
+
+
+@pytest.mark.parametrize(
+    "module,definition",
+    [
+        ("fts5", "fts5vocab('main', 'search', 'row')"),
+        ("fts4", "fts4aux('main', 'search')"),
+    ],
+)
+def test_cross_schema_vocabulary_is_unresolved(module, definition):
+    conn = sqlite3.connect(":memory:")
+    try:
+        conn.execute(f"create virtual table search using {module}(body)")
+        conn.execute(f"create virtual table temp.words using {definition}")
+        # Cross-schema ownership is not representable by the current map.
+        # The source is itself derived, so the immediate-source policy denies it.
+        assert (
+            sqlite_derived_table_dependencies(conn, schema="temp")["words"] == "words"
+        )
+    finally:
+        conn.close()
+
+
+@pytest.mark.parametrize(
+    "definition",
+    [
+        """CREATE VIRTUAL TABLE"words"USING"fts5vocab"('search', 'row')""",
+        """CREATE VIRTUAL TABLE[words]USING[fts5vocab]('search', 'row')""",
+        """CREATE VIRTUAL TABLE`words`USING`fts5vocab`('search', 'row')""",
+    ],
+)
+def test_vocabulary_quoted_token_boundaries(definition):
+    conn = sqlite3.connect(":memory:")
+    try:
+        conn.execute("create virtual table search using fts5(body)")
+        conn.execute(definition)
+        assert sqlite_derived_table_dependencies(conn)["words"] == "search"
+    finally:
+        conn.close()

tests/test_pr76_statistics_policy.py

@@ -0,0 +1,113 @@
+"""Statistics access policy and plugin replacement coverage for PR #76."""
+
+import uuid
+
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.permissions import PermissionSQL
+from datasette.resources import TableResource
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("scope", [None, "global", "database", "table", "root"])
+async def test_statistics_denied_despite_allow_rules(scope):
+    config = {"databases": {"data": {"tables": {"sqlite_stat1": {}}}}}
+    grant = {"view-table": True}
+    if scope == "global":
+        config["permissions"] = grant
+    elif scope == "database":
+        config["databases"]["data"]["permissions"] = grant
+    elif scope == "table":
+        config["databases"]["data"]["tables"]["sqlite_stat1"]["permissions"] = grant
+    ds = Datasette(memory=True, config=config)
+    ds.root_enabled = scope == "root"
+    actor = {"id": "root"} if scope == "root" else {"id": "reader"}
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.execute_write("create table items(value text)")
+    await db.execute_write("create index items_value on items(value)")
+    await db.execute_write("insert into items values ('example')")
+    await db.execute_write("analyze")
+    await ds.invoke_startup()
+    try:
+        assert "view-sqlite-statistics" not in ds.actions
+        for name in ("sqlite_stat1", "SQLITE_STAT1"):
+            assert not await ds.allowed(
+                action="view-table", resource=TableResource("data", name), actor=actor
+            )
+        for suffix in ("", ".json", ".csv"):
+            assert (
+                await ds.client.get(f"/data/sqlite_stat1{suffix}", actor=actor)
+            ).status_code == 403
+        resources = await ds.allowed_resources("view-table", parent="data", actor=actor)
+        assert "sqlite_stat1" not in {r.child for r in resources.resources}
+        assert "items" in {r.child for r in resources.resources}
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "table", ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"]
+)
+@pytest.mark.parametrize("default_deny", [False, True])
+async def test_statistics_names_denied(table, default_deny):
+    ds = Datasette(memory=True, default_deny=default_deny)
+    ds.root_enabled = True
+    await ds.invoke_startup()
+    try:
+        for name in (table, table.upper()):
+            assert not await ds.allowed(
+                action="view-table",
+                resource=TableResource("_memory", name),
+                actor={"id": "root"},
+            )
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_plugin_can_replace_statistics_policy():
+    class ReplacementPolicy:
+        @hookimpl
+        def permission_resources_sql(self, action, actor):
+            if action == "view-table":
+                return PermissionSQL(
+                    sql="SELECT 'data' AS parent, 'sqlite_stat1' AS child, :statistics_allowed AS allow, 'custom statistics policy' AS reason",
+                    params={"statistics_allowed": int(actor == {"id": "reader"})},
+                )
+
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.execute_write("create table items(value text)")
+    await db.execute_write("analyze")
+    await ds.invoke_startup()
+    name = "datasette.default_permissions.sqlite_statistics"
+    original = ds.pm.unregister(name=name)
+    assert original is not None
+    replacement = ReplacementPolicy()
+    ds.pm.register(replacement, name="test-replacement-statistics-policy")
+    try:
+        actor = {"id": "reader"}
+        assert await ds.allowed(
+            action="view-table",
+            resource=TableResource("data", "sqlite_stat1"),
+            actor=actor,
+        )
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "sqlite_stat1")
+        )
+        resources = await ds.allowed_resources(
+            "view-table", parent="data", actor=actor, include_is_private=True
+        )
+        stats = [r for r in resources.resources if r.child == "sqlite_stat1"]
+        assert len(stats) == 1 and stats[0].private
+        assert (
+            await ds.client.get("/data/sqlite_stat1.json", actor=actor)
+        ).status_code == 200
+        assert (await ds.client.get("/data/sqlite_stat1.json")).status_code == 403
+    finally:
+        ds.pm.unregister(replacement)
+        ds.pm.register(original, name=name)
+        ds.close()

tests/test_publish_cloudrun.py

@@ -1,10 +1,12 @@
-from click.testing import CliRunner
-from datasette import cli
-from unittest import mock
 import json
 import os
-import pytest
 import textwrap
+from unittest import mock
+
+import pytest
+from click.testing import CliRunner
+
+from datasette import cli
 
 
 @pytest.mark.serial
@@ -70,9 +72,7 @@ def test_publish_cloudrun_prompts_for_service(
             ),
             mock.call(f"gcloud builds submit --tag {tag}", shell=True),
             mock.call(
-                "gcloud run deploy --allow-unauthenticated --platform=managed --image {} input-service --max-instances 1".format(
-                    tag
-                ),
+                f"gcloud run deploy --allow-unauthenticated --platform=managed --image {tag} input-service --max-instances 1",
                 shell=True,
             ),
         ]
@@ -107,9 +107,7 @@ def test_publish_cloudrun(mock_call, mock_output, mock_which, tmp_path_factory):
             ),
             mock.call(f"gcloud builds submit --tag {tag}", shell=True),
             mock.call(
-                "gcloud run deploy --allow-unauthenticated --platform=managed --image {} test --max-instances 1".format(
-                    tag
-                ),
+                f"gcloud run deploy --allow-unauthenticated --platform=managed --image {tag} test --max-instances 1",
                 shell=True,
             ),
         ]
@@ -186,13 +184,13 @@ def test_publish_cloudrun_memory_cpu(
     tag = f"us-docker.pkg.dev/{mock_output.return_value}/datasette/datasette-test"
     expected_call = (
         "gcloud run deploy --allow-unauthenticated --platform=managed"
-        " --image {} test".format(tag)
+        f" --image {tag} test"
     )
     expected_build_call = f"gcloud builds submit --tag {tag}"
     if memory:
-        expected_call += " --memory {}".format(memory)
+        expected_call += f" --memory {memory}"
     if cpu:
-        expected_call += " --cpu {}".format(cpu)
+        expected_call += f" --cpu {cpu}"
     if timeout:
         expected_build_call += f" --timeout {timeout}"
     # max_instances defaults to 1

tests/test_publish_heroku.py

@@ -1,9 +1,11 @@
-from click.testing import CliRunner
-from datasette import cli
-from unittest import mock
 import os
 import pathlib
+from unittest import mock
+
 import pytest
+from click.testing import CliRunner
+
+from datasette import cli
 
 
 @pytest.mark.serial

tests/test_pytest_autoclose_plugin.py

@@ -20,6 +20,7 @@ def _run_pytest(tmp_path: Path) -> subprocess.CompletedProcess:
         cwd=str(tmp_path),
         capture_output=True,
         text=True,
+        check=False,
     )
 
 

tests/test_queries.py

@@ -15,37 +15,29 @@ from datasette.utils.sqlite import sqlite3, supports_returning
 requires_sqlite_returning = pytest.mark.skipif(
     not supports_returning(), reason="SQLite does not support RETURNING"
 )
-EXPECTED_CREATE_TABLE_TEMPLATE_SQL = "\n".join(
-    (
-        "create table new_table (",
-        "  id integer primary key,",
-        "  name text",
-        "  -- created text default (datetime('now'))",
-        ")",
-    )
-)
+EXPECTED_CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n  id integer primary key,\n  name text\n  -- created text default (datetime('now'))\n)"
 
 
 def _template_option_attributes(html, table):
-    match = re.search(r'<option value="{}"([^>]*)>'.format(table), html)
-    assert match, "Could not find template option for {}".format(table)
+    match = re.search(rf'<option value="{table}"([^>]*)>', html)
+    assert match, f"Could not find template option for {table}"
     return match.group(1)
 
 
 def _template_sql(html, table, operation):
     attrs = _template_option_attributes(html, table)
-    match = re.search(r'data-template-{}-sql="([^"]*)"'.format(operation), attrs)
-    assert match, "Could not find {} template for {}".format(operation, table)
+    match = re.search(rf'data-template-{operation}-sql="([^"]*)"', attrs)
+    assert match, f"Could not find {operation} template for {table}"
     return unescape(match.group(1))
 
 
 def _template_button_sql(html, operation):
     soup = Soup(html, "html.parser")
-    button = soup.select_one('button[data-sql-template="{}"]'.format(operation))
-    assert button, "Could not find {} template button".format(operation)
+    button = soup.select_one(f'button[data-sql-template="{operation}"]')
+    assert button, f"Could not find {operation} template button"
     assert button.get(
         "data-template-sql"
-    ), "Could not find SQL for {} template button".format(operation)
+    ), f"Could not find SQL for {operation} template button"
     return button["data-template-sql"]
 
 
@@ -53,10 +45,10 @@ async def add_numbered_queries(ds, database, count):
     for i in range(1, count + 1):
         await ds.add_query(
             database,
-            "demo_query_{:02d}".format(i),
-            "select {} as query_number".format(i),
-            title="Demo query {:02d}".format(i),
-            description="Seeded demo query number {:02d}".format(i),
+            f"demo_query_{i:02d}",
+            f"select {i} as query_number",
+            title=f"Demo query {i:02d}",
+            description=f"Seeded demo query number {i:02d}",
             source="user",
             owner_id="root",
         )
@@ -2431,9 +2423,7 @@ async def test_execute_write_json_returning_rows_can_be_truncated():
     db = ds.add_memory_database("execute_write_returning_json_truncated", name="data")
     await db.execute_write("create table dogs (id integer primary key, name text)")
     for index in range(1, 12):
-        await db.execute_write(
-            "insert into dogs (name) values (?)", ["Dog {}".format(index)]
-        )
+        await db.execute_write("insert into dogs (name) values (?)", [f"Dog {index}"])
     await ds.invoke_startup()
 
     response = await ds.client.post(
@@ -2448,7 +2438,7 @@ async def test_execute_write_json_returning_rows_can_be_truncated():
     assert data["message"] == "Query executed"
     assert data["rowcount"] == -1
     assert data["rows"] == [
-        {"id": index, "name": "Dog {}!".format(index)} for index in range(1, 11)
+        {"id": index, "name": f"Dog {index}!"} for index in range(1, 11)
     ]
     assert data["truncated"] is True
     assert (await db.execute("select count(*) from dogs where name like '%!'")).first()[
@@ -2502,9 +2492,7 @@ async def test_execute_write_html_returning_rows_can_be_truncated():
     db = ds.add_memory_database("execute_write_returning_html_truncated", name="data")
     await db.execute_write("create table dogs (id integer primary key, name text)")
     for index in range(1, 12):
-        await db.execute_write(
-            "insert into dogs (name) values (?)", ["Dog {}".format(index)]
-        )
+        await db.execute_write("insert into dogs (name) values (?)", [f"Dog {index}"])
     await ds.invoke_startup()
 
     response = await ds.client.post(
@@ -3260,74 +3248,6 @@ async def test_execute_write_create_table_uses_create_table_permission():
     assert not await db.table_exists("should_not_exist")
 
 
-@pytest.mark.asyncio
-async def test_execute_write_create_view_uses_create_view_permission():
-    ds = Datasette(
-        memory=True,
-        default_deny=True,
-        config={
-            "permissions": {
-                "insert-row": {"id": "row-writer"},
-                "update-row": {"id": "row-writer"},
-            },
-            "databases": {
-                "data": {
-                    "permissions": {
-                        "view-database": {"id": ["creator", "row-writer"]},
-                        "execute-write-sql": {"id": ["creator", "row-writer"]},
-                        "create-view": {"id": "creator"},
-                    }
-                }
-            },
-        },
-    )
-    db = ds.add_memory_database("execute_write_create_view", name="data")
-    await db.execute_write("create table dogs (id integer primary key, name text)")
-    await ds.invoke_startup()
-
-    analysis_response = await ds.client.get(
-        "/data/-/execute-write/analyze",
-        actor={"id": "creator"},
-        params={"sql": "create view dog_names as select id, name from dogs"},
-    )
-    allowed_response = await ds.client.post(
-        "/data/-/execute-write",
-        actor={"id": "creator"},
-        json={"sql": "create view dog_names as select id, name from dogs"},
-    )
-    row_permission_response = await ds.client.post(
-        "/data/-/execute-write",
-        actor={"id": "row-writer"},
-        json={"sql": "create view should_not_exist as select id from dogs"},
-    )
-
-    assert analysis_response.status_code == 200
-    analysis_data = analysis_response.json()
-    assert analysis_data["ok"] is True
-    assert analysis_data["execute_disabled"] is False
-    assert analysis_data["analysis_rows"] == [
-        {
-            "operation": "create",
-            "database": "data",
-            "table": "dog_names",
-            "required_permission": "create-view",
-            "source": None,
-            "allowed": True,
-        }
-    ]
-
-    assert allowed_response.status_code == 200
-    assert allowed_response.json()["ok"] is True
-    assert allowed_response.json()["message"] == "Query executed"
-    assert await db.view_exists("dog_names")
-
-    assert row_permission_response.status_code == 403
-    assert row_permission_response.json()["errors"] == [
-        "Permission denied: need create-view on data"
-    ]
-    assert not await db.view_exists("should_not_exist")
-
-
 @pytest.mark.parametrize(
     (
         "database_name",
@@ -3645,12 +3565,12 @@ async def test_private_query_restricts_broad_update_delete_permissions(
     )
 
     private_response = await ds.client.post(
-        "/data/alice_private/{}".format(path_suffix),
+        f"/data/alice_private/{path_suffix}",
         actor={"id": "bob"},
         json=request_json,
     )
     public_response = await ds.client.post(
-        "/data/alice_public/{}".format(path_suffix),
+        f"/data/alice_public/{path_suffix}",
         actor={"id": "bob"},
         json=request_json,
     )
@@ -3863,6 +3783,6 @@ async def test_stored_query_json_uses_parameters_not_params():
     assert "params" not in definition["query"]
 
     listing = (await ds.client.get("/data/-/queries.json")).json()
-    query = [q for q in listing["queries"] if q["name"] == "with_params"][0]
+    query = next(q for q in listing["queries"] if q["name"] == "with_params")
     assert query["parameters"] == ["name", "age"]
     assert "params" not in query

tests/test_restriction_sql.py

@@ -1,4 +1,5 @@
 import pytest
+
 from datasette.app import Datasette
 from datasette.permissions import PermissionSQL
 from datasette.resources import TableResource

tests/test_routes.py

@@ -1,8 +1,9 @@
-from datasette.app import Datasette, Database
-from datasette.utils import resolve_routes
 import pytest
 import pytest_asyncio
 
+from datasette.app import Database, Datasette
+from datasette.utils import resolve_routes
+
 
 @pytest.fixture(scope="session")
 def routes():

tests/test_schema_endpoints.py

@@ -1,5 +1,6 @@
 import pytest
 import pytest_asyncio
+
 from datasette.app import Datasette
 
 
@@ -245,3 +246,114 @@ async def test_table_not_exists(schema_ds):
     response = await schema_ds.client.get("/schema_public_db/nonexistent/-/schema.md")
     assert response.status_code == 404
     assert "not found" in response.text.lower()
+
+
+@pytest_asyncio.fixture(scope="module")
+async def schema_table_perms_ds():
+    """
+    A database that is viewable by anonymous users, but with one table
+    locked down using the documented per-table lockdown recipe:
+    a table-level allow block combined with allow_sql: false.
+    """
+    ds = Datasette(
+        config={
+            "databases": {
+                "schema_table_perms_db": {
+                    "allow_sql": False,
+                    "tables": {"employee_salaries": {"allow": {"id": "root"}}},
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("schema_table_perms_db")
+    await db.execute_write(
+        "CREATE TABLE IF NOT EXISTS public_posts (id INTEGER PRIMARY KEY, title TEXT)"
+    )
+    await db.execute_write(
+        "CREATE TABLE IF NOT EXISTS employee_salaries "
+        "(id INTEGER PRIMARY KEY, ssn TEXT, salary_usd INTEGER)"
+    )
+    await db.execute_write(
+        "CREATE INDEX IF NOT EXISTS idx_employee_salaries_ssn ON employee_salaries(ssn)"
+    )
+    await db.execute_write(
+        "CREATE TRIGGER IF NOT EXISTS trg_employee_salaries "
+        "AFTER INSERT ON employee_salaries BEGIN SELECT 1; END"
+    )
+    return ds
+
+
+@pytest.mark.asyncio
+async def test_schema_table_perms_controls(schema_table_perms_ds):
+    """Sanity check: the locked down table really is denied to anonymous users."""
+    ds = schema_table_perms_ds
+    for path in (
+        "/schema_table_perms_db/employee_salaries.json",
+        "/schema_table_perms_db/employee_salaries/-/schema.json",
+        "/schema_table_perms_db/-/query.json?sql=select+*+from+employee_salaries",
+    ):
+        response = await ds.client.get(path)
+        assert response.status_code == 403, path
+    response = await ds.client.get("/schema_table_perms_db.json")
+    assert response.status_code == 200
+    assert "employee_salaries" not in response.text
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "base_url",
+    ["/-/schema", "/schema_table_perms_db/-/schema"],
+)
+@pytest.mark.parametrize("format_ext", ["json", "md", ""])
+async def test_schema_parent_views_hide_denied_tables(
+    schema_table_perms_ds, base_url, format_ext
+):
+    """
+    GHSA-926p-cw2f-643h: /-/schema and /db/-/schema must not disclose the DDL
+    of tables the actor is denied view-table on, including indexes and
+    triggers that belong to those tables.
+    """
+    url = base_url + (f".{format_ext}" if format_ext else "")
+
+    # Anonymous: allowed table visible, denied table (and its columns,
+    # index and trigger) absent
+    response = await schema_table_perms_ds.client.get(url)
+    assert response.status_code == 200
+    assert "public_posts" in response.text
+    assert "employee_salaries" not in response.text
+    assert "ssn" not in response.text
+    assert "salary_usd" not in response.text
+    assert "idx_employee_salaries_ssn" not in response.text
+    assert "trg_employee_salaries" not in response.text
+
+    # root can see everything
+    response = await schema_table_perms_ds.client.get(url, actor={"id": "root"})
+    assert response.status_code == 200
+    assert "public_posts" in response.text
+    assert "CREATE TABLE employee_salaries" in response.text
+    assert "idx_employee_salaries_ssn" in response.text
+    assert "trg_employee_salaries" in response.text
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "object_name", ["idx_employee_salaries_ssn", "trg_employee_salaries"]
+)
+@pytest.mark.parametrize("format_ext", ["json", "md", ""])
+async def test_table_schema_does_not_serve_objects_of_denied_table(
+    schema_table_perms_ds, object_name, format_ext
+):
+    """
+    Related to GHSA-926p-cw2f-643h: /db/<name>/-/schema looks up sqlite_master
+    by name without restricting to tables/views, so requesting the name of an
+    index or trigger that belongs to a denied table serves its DDL. The
+    view-table check runs against the index/trigger name, which is not a
+    restricted table, so it passes.
+    """
+    url = f"/schema_table_perms_db/{object_name}/-/schema"
+    if format_ext:
+        url += f".{format_ext}"
+    response = await schema_table_perms_ds.client.get(url)
+    assert response.status_code in (403, 404)
+    assert "employee_salaries" not in response.text
+    assert "ssn" not in response.text

tests/test_search_tables.py

@@ -4,6 +4,7 @@ Tests for special endpoints in datasette/views/special.py
 
 import pytest
 import pytest_asyncio
+
 from datasette.app import Datasette
 
 

tests/test_shutdown.py

@@ -0,0 +1,318 @@
+"""
+Tests for the shutdown(datasette) plugin hook and Datasette.invoke_shutdown().
+
+Order under test: plugin `shutdown` hooks run first (while background tasks
+are still alive) -> supervised background tasks are cancelled and drained
+(fixed 5s grace) -> databases are closed. Hook exceptions are logged, never
+propagated, and never skip a later step. The whole sequence is idempotent,
+so a second lifespan.shutdown message (or any other second caller) is a
+no-op.
+"""
+
+import asyncio
+import contextlib
+import logging
+
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.plugins import pm
+
+
+async def _drive_lifespan(app, messages):
+    """Drive a single ASGI lifespan connection against `app`, feeding
+    `messages` to it in order via receive(). Returns once `app` itself
+    returns, which - per AsgiLifespan - happens after it has processed a
+    lifespan.shutdown message and sent back .complete or .failed. Returns
+    the list of messages `app` sent via send().
+
+    `messages` must end with a lifespan.shutdown (or a startup that fails)
+    or this will hang forever waiting for a message that never comes,
+    since AsgiLifespan only returns after handling shutdown.
+    """
+    sent = []
+    idx = 0
+
+    async def receive():
+        nonlocal idx
+        if idx < len(messages):
+            message = messages[idx]
+            idx += 1
+            return message
+        # Real servers park here waiting for lifespan.shutdown; nothing
+        # left to deliver in this test, so just block - the caller is
+        # expected to have already gotten what it needs via a preceding
+        # lifespan.shutdown in `messages`.
+        await asyncio.Event().wait()
+
+    async def send(message):
+        sent.append(message)
+
+    await app({"type": "lifespan"}, receive, send)
+    return sent
+
+
+@pytest.mark.asyncio
+async def test_shutdown_hook_runs_before_task_cancellation_then_closes():
+    # Ordering proof: the shutdown hook observes the background task still
+    # "running" (hooks run BEFORE cancellation); after the whole lifespan
+    # drive completes, the task is "cancelled" and the Datasette instance
+    # is closed.
+    events = []
+    task_state_seen_in_hook = {}
+
+    async def bg_task(datasette):
+        await asyncio.Event().wait()
+
+    class LifecyclePlugin:
+        __name__ = "LifecyclePlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(bg_task, name="bg-task")
+
+            return inner
+
+        @hookimpl
+        def shutdown(self, datasette):
+            async def inner():
+                events.append("shutdown-hook-ran")
+                handle = datasette._background_tasks.tasks()[0]
+                task_state_seen_in_hook["state"] = handle.state
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(LifecyclePlugin(), name="lifecycle_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan(
+            app, [{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]
+        )
+    finally:
+        pm.unregister(name="lifecycle_plugin")
+
+    assert {"type": "lifespan.startup.complete"} in messages
+    assert {"type": "lifespan.shutdown.complete"} in messages
+    assert events == ["shutdown-hook-ran"]
+    assert task_state_seen_in_hook["state"] == "running"
+
+    handle = ds._background_tasks.tasks()[0]
+    assert handle.state == "cancelled"
+    assert ds._closed is True
+    assert ds._shutdown_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_second_lifespan_shutdown_does_not_double_invoke():
+    call_count = {"n": 0}
+
+    class CountingShutdownPlugin:
+        __name__ = "CountingShutdownPlugin"
+
+        @hookimpl
+        def shutdown(self, datasette):
+            async def inner():
+                call_count["n"] += 1
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(CountingShutdownPlugin(), name="counting_shutdown_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan(
+            app, [{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]
+        )
+        assert {"type": "lifespan.shutdown.complete"} in messages
+        assert call_count["n"] == 1
+
+        # A second, separate lifespan connection (a misbehaving host, or a
+        # second embedder driving the same Datasette instance) sends
+        # lifespan.shutdown again. AsgiLifespan itself has no memory of
+        # the earlier connection, so this exercises invoke_shutdown()'s
+        # own idempotency guard, not anything ASGI-layer.
+        messages2 = await _drive_lifespan(app, [{"type": "lifespan.shutdown"}])
+        assert {"type": "lifespan.shutdown.complete"} in messages2
+        assert call_count["n"] == 1
+    finally:
+        pm.unregister(name="counting_shutdown_plugin")
+
+    assert ds._closed is True
+
+
+@pytest.mark.asyncio
+async def test_raising_shutdown_hook_is_logged_and_does_not_block_the_rest(caplog):
+    events = []
+
+    async def bg_task(datasette):
+        await asyncio.Event().wait()
+
+    class RaisingShutdownPlugin:
+        __name__ = "RaisingShutdownPlugin"
+
+        @hookimpl
+        def shutdown(self, datasette):
+            async def inner():
+                raise RuntimeError("boom from shutdown hook")
+
+            return inner
+
+    class WellBehavedPlugin:
+        __name__ = "WellBehavedPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(bg_task, name="bg-task")
+
+            return inner
+
+        @hookimpl
+        def shutdown(self, datasette):
+            async def inner():
+                events.append("well-behaved-ran")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(RaisingShutdownPlugin(), name="raising_shutdown_plugin")
+    pm.register(WellBehavedPlugin(), name="well_behaved_plugin")
+    try:
+        app = ds.app()
+        with caplog.at_level(logging.ERROR, logger="datasette"):
+            messages = await _drive_lifespan(
+                app, [{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]
+            )
+    finally:
+        pm.unregister(name="raising_shutdown_plugin")
+        pm.unregister(name="well_behaved_plugin")
+
+    # The exception must never turn into lifespan.shutdown.failed - it's
+    # swallowed inside invoke_shutdown(), logged, and teardown continues.
+    assert {"type": "lifespan.shutdown.complete"} in messages
+    assert events == ["well-behaved-ran"]
+    assert "shutdown hook failed" in caplog.text
+    assert "boom from shutdown hook" in caplog.text
+
+    handle = ds._background_tasks.tasks()[0]
+    assert handle.state == "cancelled"
+    assert ds._closed is True
+
+
+@pytest.mark.asyncio
+async def test_sync_shutdown_hook_variant_works():
+    events = []
+
+    class SyncShutdownPlugin:
+        __name__ = "SyncShutdownPlugin"
+
+        @hookimpl
+        def shutdown(self, datasette):
+            # Deliberately not returning a coroutine/callable - a plain
+            # sync hookimpl, same as `def startup(datasette): ...` is
+            # supported via await_me_maybe.
+            events.append("sync-shutdown-ran")
+
+    ds = Datasette(memory=True)
+    pm.register(SyncShutdownPlugin(), name="sync_shutdown_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan(
+            app, [{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]
+        )
+    finally:
+        pm.unregister(name="sync_shutdown_plugin")
+
+    assert {"type": "lifespan.shutdown.complete"} in messages
+    assert events == ["sync-shutdown-ran"]
+    assert ds._closed is True
+
+
+@pytest.mark.asyncio
+async def test_shutdown_logs_stragglers_that_outlive_the_grace_period(
+    caplog, monkeypatch
+):
+    async def stubborn(datasette):
+        with contextlib.suppress(asyncio.CancelledError):
+            await asyncio.sleep(10)
+        # Swallowing CancelledError above and returning normally simulates
+        # a task that ignores cancellation for longer than the grace
+        # period - same shape as test_background_tasks.py's equivalent
+        # test.
+        await asyncio.sleep(10)
+
+    class StubbornTaskPlugin:
+        __name__ = "StubbornTaskPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(stubborn, name="stubborn-task")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(StubbornTaskPlugin(), name="stubborn_task_plugin")
+
+    # invoke_shutdown() calls self._background_tasks.cancel_all(grace=5.0)
+    # with a grace hardcoded in app.py, per the ticket. Rather than
+    # sleeping for a real 5s in this test, monkeypatch the
+    # BackgroundTaskSupervisor *instance's* cancel_all to a wrapper that
+    # ignores the caller-supplied grace and substitutes a small one - this
+    # is the cleanest seam because it requires no production-code changes
+    # (no grace= setting/attribute to add) and leaves invoke_shutdown's
+    # own code under test untouched.
+    real_cancel_all = ds._background_tasks.cancel_all
+
+    async def fast_cancel_all(grace=5.0):
+        return await real_cancel_all(grace=0.1)
+
+    monkeypatch.setattr(ds._background_tasks, "cancel_all", fast_cancel_all)
+
+    try:
+        app = ds.app()
+
+        sent = []
+        queue = asyncio.Queue()
+        startup_complete = asyncio.Event()
+
+        async def receive():
+            return await queue.get()
+
+        async def send(message):
+            sent.append(message)
+            if message.get("type") == "lifespan.startup.complete":
+                startup_complete.set()
+
+        task = asyncio.create_task(app({"type": "lifespan"}, receive, send))
+        await queue.put({"type": "lifespan.startup"})
+        await asyncio.wait_for(startup_complete.wait(), timeout=5)
+
+        # Let the stubborn background task actually start running and
+        # reach its CancelledError-suppressing sleep before shutdown
+        # cancels it - a task cancelled before it has ever run its first
+        # step never enters that block at all, so it would finish
+        # cancelling immediately instead of behaving like a straggler.
+        await asyncio.sleep(0.05)
+
+        with caplog.at_level(logging.WARNING, logger="datasette.background_tasks"):
+            await queue.put({"type": "lifespan.shutdown"})
+            await asyncio.wait_for(task, timeout=5)
+
+        assert {"type": "lifespan.shutdown.complete"} in sent
+        assert "stubborn-task" in caplog.text
+        assert ds._closed is True
+    finally:
+        pm.unregister(name="stubborn_task_plugin")
+        # Clean up: the stubborn task ignored cancellation and is still
+        # sleeping past the shrunk grace period; actually cancel and await
+        # it now that the test has made its assertions, so it doesn't leak
+        # past the end of the test.
+        handles = ds._background_tasks.tasks()
+        if handles and handles[0].task and not handles[0].task.done():
+            handles[0].task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await handles[0].task

tests/test_spatialite.py

@@ -1,8 +1,10 @@
-from datasette.app import Datasette
-from datasette.utils import find_spatialite, SpatialiteNotFound, SPATIALITE_FUNCTIONS
-from .utils import has_load_extension
 import pytest
 
+from datasette.app import Datasette
+from datasette.utils import SPATIALITE_FUNCTIONS, SpatialiteNotFound, find_spatialite
+
+from .utils import has_load_extension
+
 
 def has_spatialite():
     try:

tests/test_stored_queries.py

@@ -1,8 +1,10 @@
-from bs4 import BeautifulSoup as Soup
-from asgiref.sync import async_to_sync
 import json
-import pytest
 import re
+
+import pytest
+from asgiref.sync import async_to_sync
+from bs4 import BeautifulSoup as Soup
+
 from .fixtures import make_app_client
 
 
@@ -206,11 +208,12 @@ def test_custom_params(stored_write_client):
     )
 
 
-def test_stored_query_pages_no_vary_header(stored_write_client):
-    # These pages no longer embed per-cookie CSRF tokens, so they must not
-    # set Vary: Cookie - they should be cacheable across users.
-    assert "vary" not in stored_write_client.get("/data").headers
-    assert "vary" not in stored_write_client.get("/data/update_name").headers
+def test_stored_query_pages_vary_by_credentials(stored_write_client):
+    # Even without per-cookie CSRF tokens, anonymous pages must not be reused
+    # for authenticated users whose permissions or navigation can differ.
+    for path in ("/data", "/data/update_name"):
+        response = stored_write_client.get(path)
+        assert response.headers["vary"] == "Cookie, Authorization"
 
 
 def test_json_post_body(stored_write_client):

tests/test_success_envelope.py

@@ -2,13 +2,14 @@
 Tests for the canonical JSON success envelope.
 
 Every JSON object returned by a Datasette endpoint on success should include
-"ok": true. (Endpoints that return a top-level array are being converted to
-objects separately - see /-/plugins, /-/databases, /-/actions.)
+"ok": true. /-/plugins intentionally returns a top-level array instead, while
+/-/databases and /-/actions use the object envelope.
 """
 
 import pytest
+
 from datasette.app import Datasette
-from datasette.utils import sqlite3, UNSTABLE_API_MESSAGE
+from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3
 
 
 @pytest.fixture
@@ -58,6 +59,7 @@ async def test_success_object_has_ok_true(ds_client, path):
         "/-/rules.json?action=view-instance",
         "/-/check.json?action=view-instance",
         "/-/threads.json",
+        "/-/tasks.json",
     ),
 )
 async def test_permission_debug_success_has_ok_true(ds_envelope, path):
@@ -79,17 +81,16 @@ async def test_permissions_post_success_has_ok_true(ds_envelope):
 
 
 @pytest.mark.asyncio
-async def test_plugins_json_is_object(ds_client):
+async def test_plugins_json_is_array(ds_client):
     response = await ds_client.get("/-/plugins.json")
     assert response.status_code == 200
     data = response.json()
-    assert set(data.keys()) == {"ok", "plugins"}
-    assert data["ok"] is True
-    assert isinstance(data["plugins"], list)
+    assert isinstance(data, list)
+    assert all(isinstance(plugin, dict) for plugin in data)
     # ?all=1 should include Datasette's default plugins in the same shape
     response_all = await ds_client.get("/-/plugins.json?all=1")
-    all_plugins = response_all.json()["plugins"]
-    assert len(all_plugins) > len(data["plugins"])
+    all_plugins = response_all.json()
+    assert len(all_plugins) > len(data)
 
 
 @pytest.mark.asyncio

tests/test_table_api.py

@@ -1,11 +1,14 @@
-from datasette.utils import detect_json1
-from datasette.utils.sqlite import sqlite_version
-from datasette.fixtures import generate_compound_rows, generate_sortable_rows
-from .fixtures import make_app_client
 import json
-import pytest
 import urllib
 
+import pytest
+
+from datasette.fixtures import generate_compound_rows, generate_sortable_rows
+from datasette.utils import detect_json1, tilde_encode
+from datasette.utils.sqlite import sqlite_version
+
+from .fixtures import make_app_client
+
 
 @pytest.mark.asyncio
 async def test_table_json(ds_client):
@@ -616,7 +619,10 @@ def test_searchmode(table_metadata, querystring, expected_rows):
             ],
         ),
         (
-            "/fixtures/searchable_view.json?_shape=arrays&_search=weasel&_fts_table=searchable_fts&_fts_pk=pk",
+            (
+                "/fixtures/searchable_view_configured_by_metadata.json"
+                "?_shape=arrays&_search=weasel&_fts_table=searchable_fts&_fts_pk=pk"
+            ),
             [[2, "terry dog", "sara weasel", "puma"]],
         ),
     ],
@@ -686,6 +692,34 @@ async def test_table_filter_queries_multiple_of_same_type(ds_client):
     ] == response.json()["rows"]
 
 
+@pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module")
+def test_table_filters_quote_identifiers():
+    with make_app_client(
+        extra_databases={"demo.db": """
+                create table "items]bracket" (
+                    id integer primary key,
+                    "name""quote" text,
+                    "tags]bracket" text
+                );
+                insert into "items]bracket" values (1, 'Alice', '["red"]');
+            """},
+    ) as client:
+        table_path = tilde_encode("items]bracket")
+        exact_query = urllib.parse.urlencode(
+            {'name"quote__exact': "Alice", "_shape": "arrays"}
+        )
+        exact_response = client.get(f"/demo/{table_path}.json?{exact_query}")
+        assert exact_response.status == 200
+        assert exact_response.json["rows"] == [[1, "Alice", '["red"]']]
+
+        array_query = urllib.parse.urlencode(
+            {"tags]bracket__arraycontains": "red", "_shape": "arrays"}
+        )
+        array_response = client.get(f"/demo/{table_path}.json?{array_query}")
+        assert array_response.status == 200
+        assert array_response.json["rows"] == [[1, "Alice", '["red"]']]
+
+
 @pytest.mark.skipif(not detect_json1(), reason="Requires the SQLite json1 module")
 @pytest.mark.asyncio
 async def test_table_filter_json_arraycontains(ds_client):
@@ -1747,3 +1781,34 @@ async def test_next_url_included_by_default(ds_client):
         data = response.json()
     assert data["next"] is None
     assert data["next_url"] is None
+
+
+@pytest.mark.asyncio
+async def test_table_through_requires_view_table_on_through_table():
+    # GHSA-53fc-rhfg-h7qp issue 3: ?_through= runs a sub-select against the
+    # caller-supplied through table, so the actor must be allowed to view it.
+    # Otherwise it is an equality oracle over any column of a denied table.
+    from datasette.app import Datasette
+
+    ds = Datasette(
+        memory=True,
+        config={"databases": {"data": {"tables": {"salaries": {"allow": False}}}}},
+    )
+    db = ds.add_memory_database("table_through_denied", name="data")
+    await db.execute_write("create table people (id integer primary key, name text)")
+    await db.execute_write(
+        "create table salaries (id integer primary key, "
+        "person_id integer references people(id), note text)"
+    )
+    await db.execute_write("insert into people values (1, 'alice'), (2, 'bob')")
+    await db.execute_write("insert into salaries values (1, 1, 'TOPSECRET-A')")
+    await ds.invoke_startup()
+
+    # Sanity: anonymous cannot read salaries directly
+    assert (await ds.client.get("/data/salaries.json")).status_code == 403
+
+    response = await ds.client.get(
+        "/data/people.json?_shape=array"
+        '&_through={"table":"salaries","column":"note","value":"TOPSECRET-A"}'
+    )
+    assert response.status_code == 403, response.text

tests/test_table_count.py

@@ -0,0 +1,191 @@
+import pytest
+import pytest_asyncio
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.database import QueryInterrupted
+from datasette.filters import FilterArguments
+from datasette.plugins import pm
+from datasette.utils import tilde_encode
+
+
+@pytest_asyncio.fixture
+async def count_ds(tmp_path):
+    ds = Datasette(
+        [tmp_path / "counts.db"],
+        config={"permissions": {"execute-sql": {"id": "sql-user"}}},
+    )
+    db = ds.get_database()
+    await db.execute_write_script("""
+        create table numbers (id integer primary key, name text);
+        insert into numbers values (1, 'one'), (2, 'two'), (3, 'three'),
+            (4, 'four'), (5, 'five'), (6, 'six'), (7, 'seven');
+        create view number_view as select id + 0 as number from numbers;
+        create table "a/b.c" (id integer);
+        insert into "a/b.c" values (1);
+        create virtual table numbers_fts using fts5(name, content="numbers");
+        insert into numbers_fts(numbers_fts) values ('rebuild');
+    """)
+    db.count_limit = 2
+    yield ds
+    ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "table,query,count",
+    [
+        ("numbers", "", 7),
+        ("numbers", "?id__gt=1", 6),
+        ("numbers", "?id__gt=1&id__lt=5", 3),
+        ("numbers", "?id__gt=1&id__gt=3", 4),
+        ("numbers", "?id__gt=100", 0),
+        ("numbers", "?name__contains=o", 3),
+        ("numbers", "?_search=two", 1),
+        ("numbers", "?_search_name=three", 1),
+        ("numbers", "?id__gt=1&_next=5&_size=1&_sort_desc=id&_nocount=1", 6),
+        # A computed column without affinity catches string/integer casting bugs.
+        ("number_view", "?number__gt=1", 6),
+        ("a/b.c", "", 1),
+    ],
+)
+async def test_count(count_ds, table, query, count):
+    response = await count_ds.client.post(
+        f"/counts/{tilde_encode(table)}/-/count{query}"
+    )
+    assert response.status_code == 200, response.text
+    assert response.json() == {"ok": True, "count": count}
+    assert response.headers["cache-control"] == "no-store"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["get", "head", "put", "delete"])
+async def test_count_post_only(count_ds, method):
+    response = await count_ds.client.request(method, "/counts/numbers/-/count")
+    assert response.status_code == 405
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "path,status",
+    [
+        ("/missing/numbers/-/count", 404),
+        ("/counts/missing/-/count", 404),
+        ("/counts/numbers/-/count?_fts_table=invalid", 400),
+        ("/counts/numbers/-/count?_where=id>1", 403),
+    ],
+)
+async def test_count_errors(count_ds, path, status):
+    response = await count_ds.client.post(
+        path, json={}, headers={"Accept": "application/json"}
+    )
+    assert response.status_code == status, response.text
+    assert response.json()["ok"] is False
+
+
+@pytest.mark.asyncio
+async def test_count_permissions(tmp_path):
+    ds = Datasette(
+        [tmp_path / "counts.db"], config={"permissions": {"view-table": False}}
+    )
+    await ds.get_database().execute_write("create table secret (id integer)")
+    try:
+        response = await ds.client.post(
+            "/counts/secret/-/count", json={}, headers={"Accept": "application/json"}
+        )
+        assert response.status_code == 403
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_count_plugin_filter(count_ds):
+    class Plugin:
+        @hookimpl
+        def filters_from_request(self, request):
+            if request.args.get("_custom"):
+                return FilterArguments(["id > :custom"], params={"custom": 4})
+
+    plugin = Plugin()
+    pm.register(plugin)
+    try:
+        response = await count_ds.client.post(
+            "/counts/numbers/-/count?_custom=1",
+            json={},
+            headers={"Accept": "application/json"},
+        )
+        assert response.json() == {"ok": True, "count": 3}
+    finally:
+        pm.unregister(plugin)
+
+
+@pytest.mark.asyncio
+async def test_count_timeout(count_ds, monkeypatch):
+    db = count_ds.get_database()
+    original = db.execute
+
+    async def execute(sql, *args, **kwargs):
+        if sql.startswith("select count(*)"):
+            raise QueryInterrupted(Exception("interrupted"), sql, {})
+        return await original(sql, *args, **kwargs)
+
+    monkeypatch.setattr(db, "execute", execute)
+    response = await count_ds.client.post(
+        "/counts/numbers/-/count", json={}, headers={"Accept": "application/json"}
+    )
+    assert response.status_code == 400
+    assert response.json()["errors"] == ["Count query timed out"]
+
+
+@pytest.mark.asyncio
+async def test_count_button_without_execute_sql(count_ds):
+    from bs4 import BeautifulSoup
+
+    response = await count_ds.client.get("/counts/numbers?id__gt=1")
+    soup = BeautifulSoup(response.text, "html.parser")
+    button = soup.select_one("button.count-all")
+    assert button["data-count-url"] == "/counts/numbers/-/count"
+    assert soup.select_one(".table-count").text == "2+ rows"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "query,status,count",
+    [
+        ("?_where=id>2&_where=id<6", 200, 3),
+        ("?_where=invalid_sql(", 400, None),
+    ],
+)
+async def test_count_where(count_ds, query, status, count):
+    response = await count_ds.client.post(
+        "/counts/numbers/-/count" + query,
+        json={},
+        actor={"id": "sql-user"},
+    )
+    assert response.status_code == status
+    if status == 200:
+        assert response.json() == {"ok": True, "count": count}
+    else:
+        assert response.json()["ok"] is False
+
+
+@pytest.mark.asyncio
+async def test_count_base_url(tmp_path):
+    from bs4 import BeautifulSoup
+
+    ds = Datasette([tmp_path / "counts.db"], settings={"base_url": "/prefix/"})
+    db = ds.get_database()
+    await db.execute_write_script(
+        "create table numbers (id integer); insert into numbers values (1), (2)"
+    )
+    db.count_limit = 1
+    try:
+        response = await ds.client.get("/counts/numbers")
+        button = BeautifulSoup(response.text, "html.parser").select_one(".count-all")
+        assert button["data-count-url"] == "/prefix/counts/numbers/-/count"
+        response = await ds.client.post(
+            ds.urls.path("/counts/numbers/-/count"), json={}
+        )
+        assert response.json() == {"ok": True, "count": 2}
+    finally:
+        ds.close()

tests/test_table_html.py

@@ -1,10 +1,14 @@
+import pathlib
+import urllib.parse
+from types import SimpleNamespace
+
+import pytest
+from bs4 import BeautifulSoup as Soup
+
 from datasette.app import Datasette
 from datasette.database import Database
-from bs4 import BeautifulSoup as Soup
+
 from .fixtures import make_app_client
-import pathlib
-import pytest
-import urllib.parse
 from .utils import inner_html
 
 
@@ -12,9 +16,9 @@ def table_data_from_soup(soup):
     import json
     import re
 
-    table_script = [
+    table_script = next(
         s for s in soup.find_all("script") if "_datasetteTableData" in (s.string or "")
-    ][0]
+    )
     match = re.search(
         r"window\._datasetteTableData\s*=\s*({.*?});",
         table_script.string,
@@ -27,11 +31,11 @@ def database_data_from_soup(soup):
     import json
     import re
 
-    database_script = [
+    database_script = next(
         s
         for s in soup.find_all("script")
         if "_datasetteDatabaseData" in (s.string or "")
-    ][0]
+    )
     match = re.search(
         r"window\._datasetteDatabaseData\s*=\s*({.*?});",
         database_script.string,
@@ -69,6 +73,21 @@ DEFAULT_EXPRESSION_OPTIONS = [
 ]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("table", ("simple_primary_key", "simple_view"))
+@pytest.mark.parametrize("duration", (0.125, 0.25))
+async def test_table_footer_query_ms(ds_client, monkeypatch, table, duration):
+    times = iter((10.0, 10.0 + duration))
+    # Only mock the table view's clock, leaving SQL time limits unaffected.
+    monkeypatch.setattr(
+        "datasette.views.table.time", SimpleNamespace(perf_counter=lambda: next(times))
+    )
+    response = await ds_client.get(f"/fixtures/{table}")
+    assert response.status_code == 200
+    footer = Soup(response.text, "html.parser").find("footer")
+    assert f"Queries took {duration * 1000}ms" in footer.get_text()
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "path,expected_definition_sql",
@@ -234,7 +253,7 @@ async def test_existing_filter_redirects(ds_client):
 )
 async def test_reflected_hidden_form_fields(ds_client, qs, expected_hidden):
     # https://github.com/simonw/datasette/issues/1527
-    response = await ds_client.get("/fixtures/facetable?{}".format(qs))
+    response = await ds_client.get(f"/fixtures/facetable?{qs}")
     # In this case we should NOT have a hidden _neighborhood__exact=Downtown field
     form = Soup(response.text, "html.parser").find("form")
     hidden_inputs = {
@@ -267,7 +286,8 @@ async def test_empty_search_parameter_gets_removed(ds_client):
 async def test_searchable_view_persists_fts_table(ds_client):
     # The search form should persist ?_fts_table as a hidden field
     response = await ds_client.get(
-        "/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk"
+        "/fixtures/searchable_view_configured_by_metadata"
+        "?_fts_table=searchable_fts&_fts_pk=pk"
     )
     inputs = Soup(response.text, "html.parser").find("form").find_all("input")
     hiddens = [i for i in inputs if i["type"] == "hidden"]
@@ -394,6 +414,37 @@ async def test_sort_links(ds_client):
     ]
 
 
+@pytest.mark.asyncio
+async def test_sort_menu_excludes_unsortable_primary_key():
+    # https://github.com/simonw/datasette/issues/1980
+    ds = Datasette(
+        [],
+        metadata={
+            "databases": {
+                "data": {"tables": {"timezones": {"sortable_columns": ["tzid"]}}}
+            }
+        },
+    )
+    try:
+        db = ds.add_database(
+            Database(ds, memory_name="test_sort_menu_unsortable_pk"), name="data"
+        )
+        await db.execute_write_script("""
+            create table timezones (
+                id integer primary key,
+                tzid text
+            );
+            insert into timezones (id, tzid) values (133, 'Europe/London');
+        """)
+        response = await ds.client.get("/data/timezones")
+        assert response.status_code == 200
+        select = Soup(response.text, "html.parser").find("select", {"name": "_sort"})
+        options = [option["value"] for option in select.find_all("option")]
+        assert options == ["", "tzid"]
+    finally:
+        ds.close()
+
+
 @pytest.mark.asyncio
 async def test_facet_display(ds_client):
     response = await ds_client.get(
@@ -480,6 +531,31 @@ async def test_facet_display(ds_client):
     ]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "filters", ["state=CA", "state__exact=CA", "state=CA&state__exact=CA"]
+)
+async def test_facet_remove_exact_filter(ds_client, filters):
+    response = await ds_client.get(
+        f"/fixtures/facetable?_facet=state&{filters}&on_earth=1"
+    )
+    assert response.status_code == 200
+    soup = Soup(response.text, "html.parser")
+    remove_link = soup.select_one('.facet-info[data-column="state"] li a.cross')
+    assert remove_link is not None
+    url = urllib.parse.urlsplit(remove_link["href"])
+    assert urllib.parse.parse_qsl(url.query) == [
+        ("_facet", "state"),
+        ("on_earth", "1"),
+    ]
+    unfiltered = await ds_client.get(f"{url.path}.json?{url.query}")
+    assert unfiltered.status_code == 200
+    rows = unfiltered.json()["rows"]
+    assert len(rows) == 14
+    assert all(row["on_earth"] == 1 for row in rows)
+    assert {row["state"] for row in rows} == {"CA", "MI"}
+
+
 @pytest.mark.asyncio
 async def test_facets_persist_through_filter_form(ds_client):
     response = await ds_client.get(
@@ -664,9 +740,7 @@ async def test_table_html_no_primary_key(ds_client):
     ]
     expected = [
         [
-            '<td class="col-Link type-pk"><a href="/fixtures/no_primary_key/{}">{}</a></td>'.format(
-                i, i
-            ),
+            f'<td class="col-Link type-pk"><a href="/fixtures/no_primary_key/{i}">{i}</a></td>',
             f'<td class="col-rowid type-int">{i}</td>',
             f'<td class="col-content type-str">{i}</td>',
             f'<td class="col-a type-str">a{i}</td>',
@@ -760,6 +834,39 @@ async def test_table_html_foreign_key_links(ds_client):
     ]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("referenced_table", ("authors", "AuThOrS"))
+async def test_table_html_foreign_key_to_missing_table_is_not_linked(referenced_table):
+    # https://github.com/simonw/datasette/issues/1515
+    ds = Datasette([])
+    db = ds.add_database(
+        Database(ds, memory_name="test_foreign_key_to_missing_table"), name="data"
+    )
+    await db.execute_write_script(f"""
+        create table authors (id integer primary key, name text);
+        create table books (
+            id integer primary key,
+            author_id integer references {referenced_table}(id),
+            missing_id integer references missing_table(id)
+        );
+        insert into authors (id, name) values (1, 'Ada');
+        insert into books (id, author_id, missing_id) values (1, 1, 7);
+        """)
+    response = await ds.client.get("/data/books")
+    assert response.status_code == 200
+    table = Soup(response.text, "html.parser").find("table")
+    cells = {td["class"][0]: str(td) for td in table.select("tbody tr")[0].select("td")}
+    assert cells["col-author_id"] == (
+        '<td class="col-author_id type-int">'
+        f'<a href="/data/{referenced_table}/1">Ada</a> <em>1</em></td>'
+    )
+    assert cells["col-missing_id"] == '<td class="col-missing_id type-int">7</td>'
+    # The JSON labels are left alone as well
+    data = (await ds.client.get("/data/books.json?_labels=on")).json()
+    assert data["rows"][0]["missing_id"] == 7
+    assert data["rows"][0]["author_id"] == {"value": 1, "label": "Ada"}
+
+
 @pytest.mark.asyncio
 async def test_table_html_foreign_key_facets(ds_client):
     response = await ds_client.get(
@@ -1667,7 +1774,7 @@ async def test_row_update_sets_message():
         assert response.status_code == 200
         assert response.json()["rows"][0]["name"] == long_name
         assert ds.unsign(response.cookies["ds_messages"], "messages") == [
-            ["Updated row 1 ({})".format(truncated_name), ds.INFO]
+            [f"Updated row 1 ({truncated_name})", ds.INFO]
         ]
     finally:
         ds.close()
@@ -1680,9 +1787,9 @@ def test_table_data_uses_base_url(app_client_base_url_prefix):
     import re
 
     soup = Soup(response.text, "html.parser")
-    table_script = [
+    table_script = next(
         s for s in soup.find_all("script") if "_datasetteTableData" in (s.string or "")
-    ][0]
+    )
     match = re.search(
         r"window\._datasetteTableData\s*=\s*({.*?});",
         table_script.string,
@@ -1710,16 +1817,17 @@ def test_table_fragment_custom_table_include():
 
 @pytest.mark.asyncio
 async def test_table_fragment_uses_render_cell_hook():
-    from datasette import hookimpl
     from markupsafe import Markup
 
+    from datasette import hookimpl
+
     class TestRenderCellPlugin:
         __name__ = "TestRenderCellPlugin"
 
         @hookimpl
         def render_cell(self, value, column, table, database):
             if database == "data" and table == "items" and column == "name":
-                return Markup("<strong>{}</strong>".format(value))
+                return Markup(f"<strong>{value}</strong>")
             return None
 
     ds = Datasette(memory=True)
@@ -2258,18 +2366,16 @@ def test_allow_facet_off(allow_facet):
 )
 async def test_format_of_binary_links(size, title, length_bytes):
     ds = Datasette()
-    db_name = "binary-links-{}".format(size)
+    db_name = f"binary-links-{size}"
     db = ds.add_memory_database(db_name)
-    sql = "select zeroblob({}) as blob".format(size)
-    await db.execute_write("create table blobs as {}".format(sql))
-    response = await ds.client.get("/{}/blobs".format(db_name))
+    sql = f"select zeroblob({size}) as blob"
+    await db.execute_write(f"create table blobs as {sql}")
+    response = await ds.client.get(f"/{db_name}/blobs")
     assert response.status_code == 200
-    expected = "{}>&lt;Binary:&nbsp;{}&nbsp;bytes&gt;</a>".format(title, length_bytes)
+    expected = f"{title}>&lt;Binary:&nbsp;{length_bytes}&nbsp;bytes&gt;</a>"
     assert expected in response.text
     # And test with arbitrary SQL query too
-    sql_response = await ds.client.get(
-        "{}/-/query".format(db_name), params={"sql": sql}
-    )
+    sql_response = await ds.client.get(f"{db_name}/-/query", params={"sql": sql})
     assert sql_response.status_code == 200
     assert expected in sql_response.text
 

tests/test_table_resource_identity.py

@@ -0,0 +1,420 @@
+"""Table permission identities must agree with SQLite identifier resolution."""
+
+import uuid
+from unittest.mock import AsyncMock
+
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.default_permissions import restrictions_allow_action
+from datasette.permissions import Action, PermissionSQL, _permission_check_cache
+from datasette.resources import QueryResource, TableResource
+from datasette.utils.actions_sql import explain_permission_for_resource
+from datasette.utils.asgi import Forbidden
+from datasette.utils.permissions import gather_permission_sql_from_hooks
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["table", "view"])
+@pytest.mark.parametrize("spelling", ["Inventory", "inventory", "INVENTORY"])
+@pytest.mark.parametrize("allowed", [False, True])
+@pytest.mark.parametrize("rule_spelling", ["Inventory", "iNvEnToRy"])
+async def test_table_permission_identity(
+    kind, spelling, allowed, rule_spelling, monkeypatch
+):
+    ds = Datasette(
+        config={
+            "permissions": {"view-table": not allowed, "insert-row": not allowed},
+            "databases": {
+                "data": {
+                    "tables": {
+                        rule_spelling: {
+                            "permissions": {
+                                "view-table": allowed,
+                                "insert-row": allowed,
+                            }
+                        }
+                    }
+                }
+            },
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    cache_token = _permission_check_cache.set({})
+    try:
+        await db.execute_write(
+            "create table Inventory (id integer primary key)"
+            if kind == "table"
+            else "create view Inventory as select 1 as id"
+        )
+        await ds.invoke_startup()
+        # Identity matching needs no target-schema lookup. Derived-table
+        # permissions may still check the schema version. All spellings and
+        # API entry points should share the existing permission result cache.
+        target_execute = AsyncMock(wraps=db.execute)
+        monkeypatch.setattr(db, "execute", target_execute)
+        internal_execute = AsyncMock(wraps=ds.get_internal_database().execute)
+        monkeypatch.setattr(ds.get_internal_database(), "execute", internal_execute)
+        resource = TableResource("data", spelling)
+        assert await ds.allowed_many(
+            actions=["view-table", "insert-row"], resource=resource
+        ) == {"view-table": allowed, "insert-row": allowed}
+        assert await ds.allowed(action="view-table", resource=resource) is allowed
+        assert await ds.check_visibility(None, "view-table", resource) == (
+            allowed,
+            False,
+        )
+        if allowed:
+            await ds.ensure_permission(action="view-table", resource=resource)
+        else:
+            with pytest.raises(Forbidden):
+                await ds.ensure_permission(action="view-table", resource=resource)
+        assert resource.child == spelling  # Do not mutate caller-owned resources.
+        for variant in ("Inventory", "inventory", "INVENTORY"):
+            assert (
+                await ds.allowed(
+                    action="view-table", resource=TableResource("data", variant)
+                )
+                is allowed
+            )
+        assert internal_execute.await_count == 1
+        assert all(
+            call.args[0] == "PRAGMA schema_version"
+            for call in target_execute.await_args_list
+        )
+        assert all(key[3] == "inventory" for key in _permission_check_cache.get())
+    finally:
+        _permission_check_cache.reset(cache_token)
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_other_permission_identities_are_preserved():
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "Äpfel": {"permissions": {"view-table": False}},
+                        "Future": {"permissions": {"view-table": False}},
+                    },
+                    "queries": {
+                        "Report": {
+                            "sql": "select 1",
+                            "permissions": {"view-query": False},
+                        },
+                        "report": {
+                            "sql": "select 1",
+                            "permissions": {"view-query": True},
+                        },
+                    },
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write('create table "Äpfel" (id integer primary key)')
+        await db.execute_write('create table "äpfel" (id integer primary key)')
+        await db.execute_write("create table Report (id integer primary key)")
+        await ds.invoke_startup()
+        # SQLite folds ASCII identifier casing, not Unicode casing.
+        for name, expected in [
+            ("ÄPFEL", False),
+            ("äPFEL", True),
+            ("Future", False),
+            ("future", False),
+        ]:
+            assert (
+                await ds.allowed(
+                    action="view-table", resource=TableResource("data", name)
+                )
+                is expected
+            )
+        # Query names remain case-sensitive even when a table has the same name.
+        for name, expected in [("Report", False), ("report", True)]:
+            assert (
+                await ds.allowed(
+                    action="view-query", resource=QueryResource("data", name)
+                )
+                is expected
+            )
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("allow", [True, False, {"id": "reader"}])
+async def test_table_listings_and_explanations(allow):
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "inventory": {"permissions": {"view-table": allow}},
+                    }
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await db.execute_write("create view InventoryView as select id from Inventory")
+        await ds.invoke_startup()
+        for actor in (None, {"id": "reader"}):
+            expected = allow is True or (isinstance(allow, dict) and actor == allow)
+            explanation = await explain_permission_for_resource(
+                datasette=ds,
+                actor=actor,
+                action="view-table",
+                parent="data",
+                child="INVENTORY",
+            )
+            assert explanation["allowed"] is expected
+            assert explanation["winning_scope"] == "resource"
+            assert any(
+                "data/inventory" in rule["reason"]
+                for rule in explanation["matched_rules"]
+            )
+            page = await ds.allowed_resources(
+                "view-table",
+                actor,
+                parent="data",
+                include_is_private=True,
+                include_reasons=True,
+                limit=1,
+            )
+            resources = [resource async for resource in page.all()]
+            matching = [r for r in resources if r.child == "Inventory"]
+            assert bool(matching) is expected
+            assert len(matching) <= 1
+            if matching:
+                assert matching[0].private is isinstance(allow, dict)
+            assert any(r.child == "InventoryView" for r in resources)
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("deny_first", [True, False])
+async def test_case_variant_rules_deny_wins(deny_first):
+    rules = [("inventory", False), ("INVENTORY", True)]
+    if not deny_first:
+        rules.reverse()
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        name: {"permissions": {"view-table": allow}}
+                        for name, allow in rules
+                    }
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await ds.invoke_startup()
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "Inventory")
+        )
+        assert not (
+            await ds.allowed_resources(
+                "view-table", parent="data", include_is_private=True
+            )
+        ).resources
+        explanation = await explain_permission_for_resource(
+            datasette=ds,
+            actor=None,
+            action="view-table",
+            parent="data",
+            child="Inventory",
+        )
+        assert not explanation["allowed"]
+        assert any(
+            rule["effect"] == "allow" and not rule["decisive"]
+            for rule in explanation["matched_rules"]
+        )
+        assert any(
+            rule["effect"] == "deny" and rule["decisive"]
+            for rule in explanation["matched_rules"]
+        )
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("config_style", ["allow", "permissions"])
+@pytest.mark.parametrize("allowed", [True, False])
+async def test_case_variant_token_restrictions(config_style, allowed):
+    table_config = (
+        {"allow": allowed}
+        if config_style == "allow"
+        else {"permissions": {"view-table": allowed}}
+    )
+    ds = Datasette(
+        config={"databases": {"data": {"tables": {"Inventory": table_config}}}}
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    actor = {"id": "reader", "_r": {"r": {"data": {"inventory": ["vt"]}}}}
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await ds.invoke_startup()
+        assert restrictions_allow_action(
+            ds, actor["_r"], "view-table", ("data", "INVENTORY")
+        )
+        assert not restrictions_allow_action(
+            ds, actor["_r"], "view-table", ("Data", "Inventory")
+        )
+        assert (
+            await ds.allowed(
+                action="view-table",
+                resource=TableResource("data", "INVENTORY"),
+                actor=actor,
+            )
+            is allowed
+        )
+        page = await ds.allowed_resources("view-table", actor, parent="data")
+        assert [(r.parent, r.child) for r in page.resources] == (
+            [("data", "Inventory")] if allowed else []
+        )
+        explanation = await explain_permission_for_resource(
+            datasette=ds,
+            actor=actor,
+            action="view-table",
+            parent="data",
+            child="Inventory",
+        )
+        assert explanation["restriction_allowed"]
+        assert explanation["allowed"] is allowed
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_plugin_restriction_intersection_and_dependencies():
+    class Plugin:
+        @hookimpl
+        def register_actions(self, datasette):
+            return [
+                Action(
+                    name="inspect-inventory",
+                    description="Inspect inventory",
+                    resource_class=TableResource,
+                    also_requires="view-table",
+                )
+            ]
+
+        @hookimpl
+        def permission_resources_sql(self, action):
+            if action not in ("view-table", "inspect-inventory"):
+                return None
+            return [
+                PermissionSQL(
+                    sql="SELECT 'data' AS parent, 'INVENTORY' AS child, 1 AS allow, 'inventory grant' AS reason",
+                    restriction_sql="SELECT 'data' AS parent, 'inventory' AS child",
+                ),
+                PermissionSQL(
+                    restriction_sql="SELECT 'data' AS parent, 'InVeNtOrY' AS child"
+                ),
+            ]
+
+    ds = Datasette(default_deny=True)
+    ds.pm.register(Plugin(), name="identity-test")
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await db.execute_write("create table Other (id integer primary key)")
+        await ds.invoke_startup()
+        for action in ("view-table", "inspect-inventory"):
+            assert await ds.allowed(
+                action=action, resource=TableResource("data", "Inventory")
+            )
+            assert not await ds.allowed(
+                action=action, resource=TableResource("data", "Other")
+            )
+            resources = (
+                await ds.allowed_resources(
+                    action, parent="data", include_is_private=True
+                )
+            ).resources
+            assert [r.child for r in resources] == ["Inventory"]
+            explanation = await explain_permission_for_resource(
+                datasette=ds,
+                actor=None,
+                action=action,
+                parent="data",
+                child="Inventory",
+            )
+            assert explanation["allowed"]
+            assert all(item["allowed"] for item in explanation["restrictions"])
+    finally:
+        ds.pm.unregister(name="identity-test")
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_shared_plugin_rule_keeps_query_identity_and_original_sql():
+    shared = PermissionSQL(
+        sql="SELECT 'data' AS parent, 'Inventory' AS child, 0 AS allow, 'shared deny' AS reason"
+    )
+    original_sql = shared.sql
+
+    class Plugin:
+        @hookimpl
+        def permission_resources_sql(self, action):
+            if action in ("view-table", "view-query"):
+                return shared
+
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "queries": {
+                        "Inventory": "select 1",
+                        "inventory": "select 1",
+                    }
+                }
+            }
+        }
+    )
+    ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    ds.pm.register(Plugin(), name="identity-test")
+    try:
+        await ds.invoke_startup()
+        for _ in range(2):
+            await gather_permission_sql_from_hooks(
+                datasette=ds, actor=None, action="view-table"
+            )
+        assert shared.sql == original_sql
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "inventory")
+        )
+        assert await ds.allowed(
+            action="view-query", resource=QueryResource("data", "inventory")
+        )
+        assert not await ds.allowed(
+            action="view-query", resource=QueryResource("data", "Inventory")
+        )
+        assert await ds.allowed(
+            action="view-table", resource=TableResource("Data", "Inventory")
+        )
+        assert restrictions_allow_action(
+            ds,
+            {"r": {"data": {"Inventory": ["vq"]}}},
+            "view-query",
+            ("data", "Inventory"),
+        )
+        assert not restrictions_allow_action(
+            ds,
+            {"r": {"data": {"Inventory": ["vq"]}}},
+            "view-query",
+            ("data", "inventory"),
+        )
+    finally:
+        ds.pm.unregister(name="identity-test")
+        ds.close()

tests/test_tasks_endpoint.py

@@ -0,0 +1,122 @@
+"""
+Tests for the /-/tasks introspection endpoint.
+
+/-/tasks exposes datasette._background_tasks (see tests/test_background_tasks.py
+for the supervisor machinery itself) the same way /-/threads exposes threading
+internals: gated behind the permissions-debug permission, JSON-only.
+"""
+
+import asyncio
+import contextlib
+import functools
+
+import pytest
+
+from datasette.app import Datasette
+
+
+async def example_task(datasette):
+    pass
+
+
+class ExampleWorker:
+    async def run(self, datasette):
+        pass
+
+    async def __call__(self, datasette):
+        pass
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "func, qualified_name",
+    [
+        (example_task, "example_task"),
+        (functools.partial(example_task), "example_task"),
+        (ExampleWorker().run, "ExampleWorker.run"),
+        (ExampleWorker(), "ExampleWorker.__call__"),
+    ],
+)
+async def test_task_function_path(func, qualified_name):
+    ds = Datasette(memory=True)
+    ds.root_enabled = True
+    handle = ds.add_background_task(func, name="custom-name")
+    try:
+        response = await ds.client.get("/-/tasks.json", actor={"id": "root"})
+        assert response.status_code == 200
+        task = response.json()["tasks"][0]
+        assert task["name"] == "custom-name"
+        assert task["function"] == f"{__name__}.{qualified_name}"
+        assert handle.function == task["function"]
+        assert "plugin" not in task
+        await handle.task
+        html = await ds.client.get("/-/tasks", actor={"id": "root"})
+        assert html.status_code == 200
+        assert task["function"] in html.text
+    finally:
+        await ds.invoke_shutdown()
+
+
+@pytest.mark.asyncio
+async def test_tasks_requires_permissions_debug():
+    ds = Datasette(memory=True)
+    ds.root_enabled = True
+
+    denied = await ds.client.get("/-/tasks.json")
+    assert denied.status_code == 403
+
+    allowed = await ds.client.get("/-/tasks.json", actor={"id": "root"})
+    assert allowed.status_code == 200
+    data = allowed.json()
+    assert data["ok"] is True
+    assert "tasks" in data
+    assert "launched" in data
+
+
+@pytest.mark.asyncio
+async def test_running_and_crashed_task_states():
+    ds = Datasette(memory=True)
+    ds.root_enabled = True
+
+    async def long_running(datasette):
+        await asyncio.Event().wait()
+
+    async def crashing_task(datasette):
+        raise RuntimeError("kaboom")
+
+    long_handle = ds.add_background_task(long_running, name="long-runner")
+    crash_handle = ds.add_background_task(crashing_task, name="crashing_task")
+
+    await ds.start_background_tasks()
+
+    # Let the crashing_task run to completion and its done-callback (which sets
+    # handle.state = "crashed") actually fire before we read state back out.
+    await asyncio.wait_for(
+        asyncio.gather(crash_handle.task, return_exceptions=True), timeout=5
+    )
+    await asyncio.sleep(0)
+
+    try:
+        response = await ds.client.get("/-/tasks.json", actor={"id": "root"})
+        assert response.status_code == 200
+        data = response.json()
+        assert data["launched"] is True
+
+        by_name = {t["name"]: t for t in data["tasks"]}
+        assert by_name["long-runner"]["state"] == "running"
+        assert by_name["long-runner"]["exception"] is None
+        assert by_name["long-runner"]["started_at"] is not None
+
+        crashed = by_name["crashing_task"]
+        assert crashed["function"] == (
+            f"{__name__}.test_running_and_crashed_task_states.<locals>.crashing_task"
+        )
+        assert crashed["state"] == "crashed"
+        assert crashed["exception"] is not None
+        assert isinstance(crashed["exception"], str)
+        assert "kaboom" in crashed["exception"]
+        assert "RuntimeError" in crashed["exception"]
+    finally:
+        long_handle.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await long_handle.task

tests/test_telemetry_metrics.py

@@ -0,0 +1,483 @@
+"""
+Tests for the OpenTelemetry metrics emitted by Datasette. Gauge callbacks are
+called directly, since the pool gauges have no attributes to tell instances apart.
+"""
+
+import asyncio
+import threading
+import weakref
+
+import pytest
+
+from datasette import telemetry
+from datasette.app import Datasette
+from datasette.database import Database
+from datasette.utils.sqlite import sqlite3
+
+pytestmark = pytest.mark.filterwarnings("ignore::ResourceWarning")
+
+
+def observations(callback, datasette=None):
+    """
+    Run a gauge callback, optionally keeping only observations produced by one
+    Datasette's databases. Returns a list of (attributes dict, value).
+    """
+    results = []
+    names = None
+    if datasette is not None:
+        names = {db.name for db in telemetry._databases_of(datasette)}
+    for observation in callback():
+        attributes = dict(observation.attributes or {})
+        namespace = attributes.get("db.namespace")
+        if names is not None and namespace is not None and namespace not in names:
+            continue
+        results.append((attributes, observation.value))
+    return results
+
+
+@pytest.fixture
+def metrics_ds():
+    "A Datasette with a distinctive thread count and a uniquely named database."
+    ds = Datasette(
+        memory=True,
+        settings={"num_sql_threads": 7},
+    )
+    ds.add_memory_database("metrics_test_db")
+    try:
+        yield ds
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_sql_thread_limit_gauge_reports_num_sql_threads(metrics_ds):
+    values = [value for _, value in observations(telemetry.observe_sql_thread_limit)]
+    # Other Datasette instances may also be reporting:
+    assert 7 in values
+
+
+@pytest.mark.asyncio
+async def test_no_thread_gauges_in_non_threaded_mode():
+    "Pool gauges skip instances with num_sql_threads=0, which have no pool."
+    ds = Datasette(memory=True, settings={"num_sql_threads": 0})
+    try:
+        assert ds.executor is None
+        # Pool gauges have no attributes, so observe only this instance:
+        original = telemetry._live_datasettes
+        telemetry._live_datasettes = weakref.WeakSet([ds])
+        try:
+            assert list(telemetry.observe_sql_thread_limit()) == []
+            assert list(telemetry.observe_sql_thread_queue_depth()) == []
+        finally:
+            telemetry._live_datasettes = original
+        # Per-database gauges do not depend on the pool:
+        assert observations(telemetry.observe_pending_queries, ds)
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_thread_queue_depth_gauge_reports_saturation():
+    """
+    Queue depth is above zero when reads queue behind num_sql_threads. Also
+    fails if the private ThreadPoolExecutor._work_queue attribute goes away.
+    """
+    ds = Datasette(memory=True, settings={"num_sql_threads": 1})
+    db = ds.add_memory_database("metrics_saturation_db")
+    entered = threading.Event()
+    release = threading.Event()
+
+    def blocker(conn):
+        entered.set()
+        assert release.wait(timeout=10)
+        return 1
+
+    try:
+        first = asyncio.ensure_future(db.execute_fn(blocker))
+        # Wait until the blocker is using the only thread:
+        await asyncio.get_running_loop().run_in_executor(None, entered.wait, 10)
+        second = asyncio.ensure_future(db.execute_fn(lambda conn: 2))
+        # The second query is queued on a later event loop turn, so poll:
+        depths = []
+        for _ in range(500):
+            depths = [
+                value
+                for _, value in observations(telemetry.observe_sql_thread_queue_depth)
+            ]
+            if any(value >= 1 for value in depths):
+                break
+            await asyncio.sleep(0.01)
+        assert any(value >= 1 for value in depths), depths
+        release.set()
+        assert await first == 1
+        assert await second == 2
+    finally:
+        release.set()
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_pending_queries_gauge_tracks_in_flight_queries(metrics_ds):
+    db = metrics_ds.get_database("metrics_test_db")
+    attributes = {"db.namespace": "metrics_test_db"}
+
+    def value():
+        points = [
+            v
+            for a, v in observations(telemetry.observe_pending_queries, metrics_ds)
+            if a == attributes
+        ]
+        assert len(points) == 1
+        return points[0]
+
+    assert value() == 0
+
+    # Hold the worker thread until release is set:
+    release = asyncio.Event()
+    loop = asyncio.get_running_loop()
+    entered = asyncio.Event()
+
+    def blocking_fn(conn):
+        loop.call_soon_threadsafe(entered.set)
+        asyncio.run_coroutine_threadsafe(release.wait(), loop).result()
+        return "done"
+
+    task = asyncio.ensure_future(db.execute_fn(blocking_fn))
+    await entered.wait()
+    assert value() == 1, "a query occupying a pool thread must be counted as pending"
+    release.set()
+    assert await task == "done"
+    assert value() == 0, "the count must drop once the query completes"
+
+
+@pytest.mark.asyncio
+async def test_write_queue_depth_gauge(metrics_ds):
+    db = metrics_ds.get_database("metrics_test_db")
+    attributes = {"db.namespace": "metrics_test_db"}
+
+    def depths():
+        return [
+            v
+            for a, v in observations(telemetry.observe_write_queue_depth, metrics_ds)
+            if a == attributes
+        ]
+
+    # No observation until the write queue has been created:
+    assert depths() == []
+
+    await db.execute_write("create table t (id integer primary key)")
+    assert depths() == [0], "an idle write queue reports zero, not nothing"
+
+
+@pytest.mark.asyncio
+async def test_open_connections_gauge(metrics_ds, tmp_path):
+    path = str(tmp_path / "conns.db")
+    sqlite3.connect(path).execute("create table t (id integer primary key)")
+    db = metrics_ds.add_database(Database(metrics_ds, path=path), name="conns_db")
+    attributes = {"db.namespace": "conns_db"}
+
+    def open_connections():
+        points = [
+            v
+            for a, v in observations(telemetry.observe_open_connections, metrics_ds)
+            if a == attributes
+        ]
+        assert len(points) == 1
+        return points[0]
+
+    assert open_connections() == 0
+    await db.execute("select 1")
+    assert open_connections() >= 1, "executing a query opens a tracked connection"
+
+
+@pytest.mark.asyncio
+async def test_operation_duration_histogram_read(otel_metrics):
+    ds = Datasette(memory=True)
+    ds.add_memory_database("duration_read_db")
+    try:
+        db = ds.get_database("duration_read_db")
+        await db.execute("select 1")
+        otel_metrics.collect()
+        point = otel_metrics.point(
+            "db.client.operation.duration",
+            {"db.namespace": "duration_read_db", "datasette.operation": "read"},
+        )
+        assert point.count == 1
+        assert point.sum > 0
+        assert dict(point.attributes)["db.system"] == "sqlite"
+        assert "error.type" not in dict(point.attributes)
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_operation_duration_histogram_write(otel_metrics):
+    ds = Datasette(memory=True)
+    ds.add_memory_database("duration_write_db")
+    try:
+        db = ds.get_database("duration_write_db")
+        await db.execute_write("create table t (id integer primary key)")
+        otel_metrics.collect()
+        point = otel_metrics.point(
+            "db.client.operation.duration",
+            {"db.namespace": "duration_write_db", "datasette.operation": "write"},
+        )
+        assert point.count == 1
+        assert point.sum > 0
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_operation_duration_records_error_type(otel_metrics):
+    "A failed query is still timed, and is separable from a successful one."
+    ds = Datasette(memory=True)
+    ds.add_memory_database("duration_error_db")
+    try:
+        db = ds.get_database("duration_error_db")
+        with pytest.raises(sqlite3.OperationalError):
+            await db.execute("select * from nope")
+        otel_metrics.collect()
+        point = otel_metrics.point(
+            "db.client.operation.duration",
+            {"db.namespace": "duration_error_db", "datasette.operation": "read"},
+        )
+        assert point.count == 1
+        assert dict(point.attributes)["error.type"] == "OperationalError"
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_operation_duration_records_write_error_type(otel_metrics):
+    "A failed write is still timed and records error.type."
+    ds = Datasette(memory=True)
+    ds.add_memory_database("duration_write_error_db")
+    try:
+        db = ds.get_database("duration_write_error_db")
+        with pytest.raises(sqlite3.OperationalError):
+            await db.execute_write("insert into nope values (1)")
+        otel_metrics.collect()
+        point = otel_metrics.point(
+            "db.client.operation.duration",
+            {"db.namespace": "duration_write_error_db", "datasette.operation": "write"},
+        )
+        assert point.count == 1
+        assert dict(point.attributes)["error.type"] == "OperationalError"
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_write_queue_wait_histogram(otel_metrics):
+    ds = Datasette(memory=True)
+    ds.add_memory_database("queue_wait_db")
+    try:
+        db = ds.get_database("queue_wait_db")
+        await db.execute_write("create table t (id integer primary key)")
+        await db.execute_write("insert into t (id) values (1)")
+        otel_metrics.collect()
+        point = otel_metrics.point(
+            "datasette.write.queue_wait", {"db.namespace": "queue_wait_db"}
+        )
+        assert point.count == 2, "one measurement per write dequeued"
+        assert point.sum >= 0
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_interrupted_queries_counter(otel_metrics):
+    "Queries cancelled by sql_time_limit_ms are counted."
+    ds = Datasette(memory=True, settings={"sql_time_limit_ms": 1})
+    ds.add_memory_database("interrupted_db")
+    try:
+        db = ds.get_database("interrupted_db")
+        from datasette.database import QueryInterrupted
+
+        with pytest.raises(QueryInterrupted):
+            await db.execute("""
+                with recursive counter(x) as (
+                    select 0 union all select x + 1 from counter
+                )
+                select * from counter
+                """)
+        otel_metrics.collect()
+        point = otel_metrics.point(
+            "datasette.sql.queries.interrupted", {"db.namespace": "interrupted_db"}
+        )
+        assert point.value == 1
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_metrics_are_reported_through_the_sdk_for_gauges(otel_metrics):
+    "Gauge callbacks reach the metric reader as data points."
+    ds = Datasette(memory=True)
+    ds.add_memory_database("gauge_pipeline_db")
+    try:
+        await ds.get_database("gauge_pipeline_db").execute("select 1")
+        otel_metrics.collect()
+        point = otel_metrics.point(
+            "datasette.sql.queries.pending", {"db.namespace": "gauge_pipeline_db"}
+        )
+        assert point.value == 0
+        assert otel_metrics.points("datasette.sql.threads.limit")
+    finally:
+        ds.close()
+
+
+def test_closed_datasette_stops_being_observed():
+    ds = Datasette(memory=True)
+    ds.add_memory_database("closed_db")
+    assert observations(telemetry.observe_pending_queries, ds)
+    ds.close()
+    names = [
+        attributes.get("db.namespace")
+        for attributes, _ in observations(telemetry.observe_pending_queries)
+    ]
+    assert "closed_db" not in names
+
+
+def test_registry_holds_instances_weakly():
+    """
+    Registering an instance does not keep it alive. Uses a stand-in object
+    because an atexit handler in Database.__init__ keeps a real Datasette alive.
+    """
+    import gc
+    import weakref
+
+    class FakeDatasette:
+        pass
+
+    fake = FakeDatasette()
+    telemetry.register_datasette(fake)
+    assert fake in telemetry._live_instances()
+    ref = weakref.ref(fake)
+    del fake
+    gc.collect()
+    assert ref() is None
+    assert not any(isinstance(ds, FakeDatasette) for ds in telemetry._live_instances())
+
+
+HISTOGRAM_PROBES = [
+    # (instrument attribute on telemetry, metric name, isolating attributes)
+    (
+        "sql_operation_duration",
+        "db.client.operation.duration",
+        {"db.namespace": "bucket_probe_operation"},
+    ),
+    (
+        "write_queue_wait",
+        "datasette.write.queue_wait",
+        {"db.namespace": "bucket_probe_queue_wait"},
+    ),
+]
+
+# One value in each of six registry buckets. The SDK's default boundaries
+# would put the first five in the same bucket.
+SPREAD = [0.00005, 0.0003, 0.002, 0.03, 0.8, 7.0]
+
+
+@pytest.mark.parametrize(
+    "instrument_name,metric_name,attributes",
+    HISTOGRAM_PROBES,
+    ids=[metric for _, metric, _ in HISTOGRAM_PROBES],
+)
+def test_histograms_spread_values_across_buckets(
+    otel_metrics, instrument_name, metric_name, attributes
+):
+    """
+    The registry's bucket boundaries reach the SDK. Values are recorded
+    directly because real test query durations would all share one bucket.
+    """
+    from datasette.telemetry_registry import METRICS
+
+    metric = next(m for m in METRICS if m == metric_name)
+    instrument = getattr(telemetry, instrument_name)
+    for value in SPREAD:
+        instrument.record(value, attributes)
+
+    otel_metrics.collect()
+    point = otel_metrics.point(metric_name, attributes)
+
+    assert (
+        tuple(point.explicit_bounds) == metric.buckets
+    ), "the registry's boundaries did not reach the SDK"
+    assert point.count == len(SPREAD)
+    occupied = [count for count in point.bucket_counts if count]
+    assert len(occupied) == len(SPREAD), (
+        f"expected each of {SPREAD} in its own bucket, got bucket counts "
+        f"{list(point.bucket_counts)} for bounds {list(point.explicit_bounds)}"
+    )
+
+
+@pytest.mark.asyncio
+async def test_operation_duration_histogram_records_execute_fn(otel_metrics):
+    "execute_fn() reads are recorded in the same histogram as SQL reads."
+    ds = Datasette(memory=True)
+    ds.add_memory_database("duration_fn_db")
+    try:
+        db = ds.get_database("duration_fn_db")
+
+        def read_one(conn):
+            return conn.execute("select 1").fetchone()[0]
+
+        assert await db.execute_fn(read_one) == 1
+        otel_metrics.collect()
+        point = otel_metrics.point(
+            "db.client.operation.duration",
+            {"db.namespace": "duration_fn_db", "datasette.operation": "read"},
+        )
+        assert point.count == 1
+        assert point.sum > 0
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_operation_duration_histogram_records_execute_write_fn(otel_metrics):
+    "execute_write_fn() writes are recorded in the same histogram."
+    ds = Datasette(memory=True)
+    ds.add_memory_database("duration_write_fn_db")
+    try:
+        db = ds.get_database("duration_write_fn_db")
+
+        def create_table(conn):
+            conn.execute("create table t (id integer primary key)")
+
+        await db.execute_write_fn(create_table)
+        otel_metrics.collect()
+        point = otel_metrics.point(
+            "db.client.operation.duration",
+            {"db.namespace": "duration_write_fn_db", "datasette.operation": "write"},
+        )
+        assert point.count == 1
+        assert point.sum > 0
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_operation_duration_records_callback_error_type(otel_metrics):
+    "A callback that raises is still timed, with error.type from the exception."
+    ds = Datasette(memory=True)
+    ds.add_memory_database("duration_fn_error_db")
+    try:
+        db = ds.get_database("duration_fn_error_db")
+
+        def boom(conn):
+            raise ValueError("callback failed")
+
+        with pytest.raises(ValueError):
+            await db.execute_fn(boom)
+        otel_metrics.collect()
+        point = otel_metrics.point(
+            "db.client.operation.duration",
+            {"db.namespace": "duration_fn_error_db", "datasette.operation": "read"},
+        )
+        assert point.count == 1
+        assert dict(point.attributes)["error.type"] == "ValueError"
+    finally:
+        ds.close()

tests/test_telemetry_registry.py

@@ -0,0 +1,502 @@
+"""
+Tests that the spans, attributes and metrics Datasette emits match
+datasette/telemetry_registry.py, in both directions.
+"""
+
+import copy
+import io
+import itertools
+import pickle
+
+import pytest
+import pytest_asyncio
+
+pytest.importorskip("opentelemetry.sdk")
+
+from opentelemetry.trace import SpanKind
+
+from datasette import hookimpl
+from datasette import telemetry_registry as reg
+from datasette.app import Datasette
+from datasette.database import QueryInterrupted
+from datasette.telemetry_testing import assert_metrics_conform, assert_metrics_covered
+from datasette.utils.sqlite import sqlite3
+
+# Written out as literals rather than read from the registry, so renaming a
+# signal fails these tests.
+EXPECTED_ATTRIBUTES = {
+    "db.query": {
+        "db.system",
+        "db.namespace",
+        "db.query.text",
+        "datasette.callback",
+        "db.operation.name",
+        "datasette.param_count",
+        "datasette.param_sets",
+        "datasette.time_limit_ms",
+        "datasette.rows_returned",
+        "datasette.truncated",
+        "datasette.interrupted",
+        "datasette.sql_error_suppressed",
+        "datasette.executescript",
+        "datasette.executemany",
+    },
+    "db.query.execute": set(),
+    "db.write.queue_wait": set(),
+    "db.write.execute": {
+        "datasette.isolated_connection",
+        "datasette.transaction",
+    },
+    "datasette.startup": set(),
+}
+EXPECTED_SPANS = set(EXPECTED_ATTRIBUTES)
+
+# The HTTP request span name is composed at runtime as "{method} {route}", so
+# it is checked by shape rather than as a literal. The workload only issues GETs.
+EXPECTED_HTTP_SPAN_NAME = "{http.request.method} {http.route}"
+EXPECTED_HTTP_METHOD_NAMES = {"GET"}
+EXPECTED_HTTP_ATTRIBUTES = {
+    "http.request.method",
+    "http.route",
+    "url.path",
+    "url.scheme",
+    "server.address",
+    "user_agent.original",
+    "http.response.status_code",
+    "error.type",
+    "datasette.internal_client",
+}
+
+# The registry uses the name template for the request span.
+EXPECTED_REGISTRY_ATTRIBUTES = dict(
+    EXPECTED_ATTRIBUTES, **{EXPECTED_HTTP_SPAN_NAME: EXPECTED_HTTP_ATTRIBUTES}
+)
+EXPECTED_REGISTRY_NAMES = set(EXPECTED_REGISTRY_ATTRIBUTES)
+
+# Named in-memory databases are shared between instances, so each workload
+# uses a unique name.
+_names = itertools.count()
+
+
+def _unique(prefix):
+    return f"{prefix}{next(_names)}"
+
+
+class _BoomPlugin:
+    "A route that raises, producing a 500 and error.type on the request span."
+
+    __name__ = "TelemetryRegistryBoomPlugin"
+
+    @hookimpl
+    def register_routes(self):
+        return [(r"^/-/telemetry-registry-boom$", lambda: 1 / 0)]
+
+
+async def exercise():
+    """
+    Drive enough of Datasette to emit every registered span and attribute,
+    including datasette.startup. Returns the instance so the caller can close it.
+    """
+    name = _unique("registry")
+    ds = Datasette(memory=True)
+    ds.add_memory_database(name)
+    # datasette.startup
+    await ds.invoke_startup()
+    db = ds.get_database(name)
+
+    # Writes: db.write.queue_wait, db.write.execute, db.query
+    await db.execute_write("create table t (id integer primary key, v text)")
+    # datasette.executemany, datasette.param_sets
+    await db.execute_write_many(
+        "insert into t (id, v) values (?, ?)", [[i, f"v{i}"] for i in range(30)]
+    )
+    # datasette.executescript
+    await db.execute_write_script("create table t2 (id integer); drop table t2;")
+    # datasette.transaction=False - VACUUM cannot run inside a transaction
+    await db.execute_write("vacuum", transaction=False)
+    # datasette.isolated_connection=True
+    await db.execute_isolated_fn(lambda conn: conn.execute("select 1").fetchone())
+
+    # datasette.callback, using named functions rather than lambdas
+    def registry_read_callback(conn):
+        return conn.execute("select count(*) from t").fetchone()
+
+    def registry_write_callback(conn):
+        conn.execute("insert into t (id, v) values (100, 'callback')")
+
+    await db.execute_fn(registry_read_callback)
+    await db.execute_write_fn(registry_write_callback)
+
+    # Reads: db.query.execute, datasette.rows_returned, datasette.truncated,
+    # datasette.param_count, datasette.time_limit_ms
+    await db.execute("select * from t where id > :n", {"n": 5})
+    await db.execute("select * from t", truncate=True)
+
+    # datasette.sql_error_suppressed
+    with pytest.raises(sqlite3.OperationalError):
+        await db.execute("select nope from t", log_sql_errors=False)
+
+    # datasette.interrupted: an unbounded recursive CTE always exceeds 1ms
+    with pytest.raises(QueryInterrupted):
+        await db.execute(
+            "with recursive c(x) as (select 0 union all select x+1 from c) "
+            "select * from c",
+            custom_time_limit=1,
+        )
+
+    # HTTP request spans and their attributes
+    assert (await ds.client.get(f"/{name}/t?_facet=v")).status_code == 200
+    assert (await ds.client.get(f"/{name}/t/1.json")).status_code == 200
+
+    # error.type on the request span, set by a 5xx response
+    ds.pm.register(_BoomPlugin(), name="telemetry-registry-boom")
+    try:
+        response = await ds.client.get("/-/telemetry-registry-boom")
+        assert response.status_code == 500
+    finally:
+        ds.pm.unregister(name="telemetry-registry-boom")
+    return ds
+
+
+@pytest_asyncio.fixture
+async def emitted(otel_spans):
+    """
+    Every (span name, span kind, attributes) triple emitted by exercise().
+    The kind is needed to resolve the dynamically named request span.
+    """
+    ds = await exercise()
+    spans = otel_spans.get_finished_spans()
+    assert spans, "no spans captured - the fixture is not exercising anything"
+    # str() so failure messages show plain strings, not registry instances
+    collected = tuple(
+        (
+            str(span.name),
+            span.kind,
+            {str(key): value for key, value in (span.attributes or {}).items()},
+        )
+        for span in spans
+    )
+    ds.close()
+    return collected
+
+
+def _partition(emitted):
+    "The statically named spans, and the dynamically named request spans."
+    static = [record for record in emitted if record[1] is not SpanKind.SERVER]
+    server = [record for record in emitted if record[1] is SpanKind.SERVER]
+    return static, server
+
+
+def _keys_by_span(records):
+    by_span = {}
+    for name, _kind, attributes in records:
+        by_span.setdefault(name, set()).update(attributes)
+    return by_span
+
+
+@pytest.mark.asyncio
+async def test_workload_emits_exactly_the_expected_names(emitted):
+    "Emitted span and attribute names match the expected literals."
+    static, server = _partition(emitted)
+    by_span = _keys_by_span(static)
+    assert set(by_span) == EXPECTED_SPANS
+    assert by_span == EXPECTED_ATTRIBUTES
+
+    assert server, "the workload made HTTP requests but no SERVER span was emitted"
+    union = set()
+    methods = set()
+    for name, _kind, attributes in server:
+        union |= set(attributes)
+        route = attributes.get("http.route")
+        # Every request in the workload matches a route
+        assert route, f"the request span {name!r} carries no http.route"
+        method, _, name_route = name.partition(" ")
+        assert name_route == route, (
+            f"the request span is named {name!r}, which is not the "
+            f"`{{method}} {{route}}` of {method!r} and {route!r}"
+        )
+        methods.add(method)
+    assert methods == EXPECTED_HTTP_METHOD_NAMES
+    assert union == EXPECTED_HTTP_ATTRIBUTES
+
+
+def test_registry_matches_the_expected_names():
+    "Registry names match the expected literals."
+    assert {str(span) for span in reg.SPANS} == EXPECTED_REGISTRY_NAMES
+    for span in reg.SPANS:
+        assert {
+            str(attribute) for attribute in span.attributes
+        } == EXPECTED_REGISTRY_ATTRIBUTES[str(span)], f"{span} attributes have drifted"
+
+
+@pytest.mark.asyncio
+async def test_every_emitted_span_is_registered(emitted):
+    "A span added without a registry entry would be missing from the docs."
+    unregistered = sorted(
+        {name for name, kind, _ in emitted if reg.span_for(name, kind) is None}
+    )
+    assert (
+        not unregistered
+    ), f"these spans are emitted but not in telemetry_registry.SPANS: {unregistered}"
+
+
+@pytest.mark.asyncio
+async def test_every_emitted_attribute_is_registered(emitted):
+    "An attribute added without a registry entry would be missing from the docs."
+    unregistered = sorted(
+        {
+            f"{name} -> {key}"
+            for name, kind, keys in emitted
+            for key in keys
+            if not reg.attribute_allowed(reg.span_for(name, kind), key)
+        }
+    )
+    assert (
+        not unregistered
+    ), "these span attributes are emitted but not registered: " + ", ".join(
+        unregistered
+    )
+
+
+@pytest.mark.asyncio
+async def test_every_registered_span_is_emitted(emitted):
+    "The docs should not describe a span that is no longer emitted."
+    # Compare by identity: the request span's registry name never appears on
+    # the wire.
+    resolved = {id(reg.span_for(name, kind)) for name, kind, _ in emitted}
+    missing = sorted(str(span) for span in reg.SPANS if id(span) not in resolved)
+    assert not missing, (
+        f"these spans are documented but never emitted by the workload: {missing}. "
+        "Either the instrumentation was removed, or exercise() no longer reaches it."
+    )
+
+
+@pytest.mark.asyncio
+async def test_every_registered_attribute_is_emitted(emitted):
+    """
+    Every registered attribute, including optional ones, is emitted at least
+    once. If a new attribute only appears in rare cases, extend exercise().
+    """
+    by_entry = {}
+    for name, kind, keys in emitted:
+        entry = reg.span_for(name, kind)
+        if entry is not None:
+            by_entry.setdefault(id(entry), set()).update(keys)
+    missing = []
+    for span in reg.SPANS:
+        emitted_keys = by_entry.get(id(span), set())
+        for attribute in span.attributes:
+            if attribute not in emitted_keys:
+                missing.append(f"{span} -> {attribute}")
+    assert not missing, (
+        "these attributes are documented but never emitted by the workload: "
+        + ", ".join(sorted(missing))
+    )
+
+
+def test_registry_has_no_duplicate_names():
+    assert len(set(reg.SPANS)) == len(reg.SPANS)
+    for span in reg.SPANS:
+        assert len(set(span.attributes)) == len(
+            span.attributes
+        ), f"{span} lists an attribute twice"
+
+
+def test_registry_entries_are_documented():
+    "Every entry has a description, used to generate the docs."
+    for span in reg.SPANS:
+        assert span.description.strip(), f"{span} has no description"
+        for attribute in span.attributes:
+            assert attribute.description.strip(), f"{span} -> {attribute} has none"
+
+
+def test_registry_entries_are_usable_as_plain_strings():
+    assert isinstance(reg.DB_QUERY, str)
+    assert isinstance(reg.DB_NAMESPACE, str)
+    assert reg.DB_QUERY == "db.query"
+    assert reg.DB_NAMESPACE == "db.namespace"
+    assert f"{reg.DB_QUERY}.execute" == "db.query.execute"
+
+
+def test_registry_entries_survive_deepcopy_and_pickle():
+    """
+    A copied or unpickled entry is a plain str. ConsoleMetricExporter
+    deepcopies metric attributes, which use registry entries as keys.
+    """
+    for entry in (reg.DB_NAMESPACE, reg.DB_QUERY, reg.M_OPERATION_DURATION):
+        assert copy.deepcopy({entry: 1}) == {str(entry): 1}
+        assert type(copy.deepcopy(entry)) is str
+        assert pickle.loads(pickle.dumps(entry)) == str(entry)
+        # The original entry keeps its metadata
+        assert entry.description.strip()
+
+
+@pytest.mark.asyncio
+async def test_console_metric_exporter_renders_core_metric_points(otel_metrics):
+    from opentelemetry.sdk.metrics.export import (
+        ConsoleMetricExporter,
+        MetricExportResult,
+    )
+
+    name = _unique("registry_console_export")
+    ds = Datasette(memory=True)
+    ds.add_memory_database(name)
+    await ds.invoke_startup()
+    # Produces a db.client.operation.duration point keyed by DB_NAMESPACE
+    await ds.get_database(name).execute("select 1")
+
+    data = otel_metrics.reader.get_metrics_data()
+    assert data is not None, "no metrics captured - nothing to export"
+    exporter = ConsoleMetricExporter(out=io.StringIO())
+    assert exporter.export(data) is MetricExportResult.SUCCESS
+    ds.close()
+
+
+def test_every_histogram_declares_bucket_boundaries():
+    """
+    Every histogram declares bucket boundaries, and only histograms do.
+    OpenTelemetry's defaults are meant for milliseconds, not seconds.
+    """
+    for metric in reg.METRICS:
+        if metric.kind == reg.HISTOGRAM:
+            assert metric.buckets, f"{metric} is a histogram with no boundaries"
+            assert list(metric.buckets) == sorted(
+                set(metric.buckets)
+            ), f"{metric} boundaries must be ascending and unique"
+            assert metric.buckets[0] > 0, f"{metric} has a non-positive boundary"
+        else:
+            assert (
+                metric.buckets is None
+            ), f"{metric} is a {metric.kind} and cannot have bucket boundaries"
+
+
+def test_dynamic_span_lookup():
+    """
+    dynamic=True entries such as the request span match on kind. They never
+    match without a kind, and never override a registered name.
+    """
+    assert reg.span_for("GET", SpanKind.SERVER) is reg.HTTP_REQUEST
+    assert reg.span_for("POST /^/(?P<database>[^/]+)$", SpanKind.SERVER) is (
+        reg.HTTP_REQUEST
+    )
+    assert reg.span_for("GET") is None
+    assert reg.span_for("anything at all", SpanKind.INTERNAL) is None
+    assert reg.span_for("db.query", SpanKind.SERVER) is reg.DB_QUERY
+
+
+def test_span_and_attribute_lookup():
+    assert reg.span_for("db.query") is reg.DB_QUERY
+    assert reg.span_for("datasette.startup") is reg.STARTUP
+    assert reg.span_for("not.a.datasette.span") is None
+    assert reg.attribute_allowed(reg.DB_QUERY, "db.namespace")
+    assert not reg.attribute_allowed(reg.DB_QUERY, "db.namespace.extra")
+    assert not reg.attribute_allowed(reg.DB_QUERY, "datasette.isolated_connection")
+    assert not reg.attribute_allowed(None, "db.namespace")
+
+
+# --- Metric conformance ----------------------------------------------------
+
+
+@pytest_asyncio.fixture
+async def emitted_metrics(otel_metrics):
+    """
+    Metric names and (metric name, attribute key) pairs from a broad workload.
+    Checks use attribute keys rather than values, since other Datasette
+    instances in the session can also report points.
+    """
+    # Reaches every synchronous metric except datasette.sql.queries.interrupted
+    ds = await exercise()
+
+    # datasette.sql.queries.interrupted ignores custom_time_limit timeouts, so
+    # this needs an instance with a low sql_time_limit_ms.
+    slow_name = _unique("registry_metrics_slow")
+    slow = Datasette(memory=True, settings={"sql_time_limit_ms": 5})
+    slow.add_memory_database(slow_name)
+    await slow.invoke_startup()
+    slow_db = slow.get_database(slow_name)
+    with pytest.raises(QueryInterrupted):
+        await slow_db.execute(
+            "with recursive c(x) as (select 0 union all select x+1 from c) "
+            "select * from c"
+        )
+
+    # Collect before closing the instances so the observable gauges report them
+    otel_metrics.collect()
+    snapshot = otel_metrics.snapshot
+    assert snapshot, "no metrics captured - the fixture is not exercising anything"
+    pairs = set()
+    for metric_name, points in snapshot.items():
+        for point in points:
+            for key in point.attributes or {}:
+                pairs.add((metric_name, key))
+    ds.close()
+    slow.close()
+    return {"names": set(snapshot), "pairs": pairs, "collector": otel_metrics}
+
+
+@pytest.mark.asyncio
+async def test_metrics_conform_to_the_registry(emitted_metrics):
+    """
+    Emitted metric names, kinds, units, attribute keys and enum values match
+    the registry, using the plugin testing helper.
+    """
+    assert_metrics_conform(
+        reg.METRICS, emitted_metrics["collector"], scope_name="datasette"
+    )
+
+
+@pytest.mark.asyncio
+async def test_every_registered_metric_is_emitted(emitted_metrics):
+    assert_metrics_covered(
+        reg.METRICS, emitted_metrics["collector"], scope_name="datasette"
+    )
+
+
+@pytest.mark.asyncio
+async def test_every_registered_metric_attribute_is_emitted(emitted_metrics):
+    "Every registered metric attribute, including optional ones, is emitted."
+    emitted_keys_by_metric = {}
+    for metric_name, key in emitted_metrics["pairs"]:
+        emitted_keys_by_metric.setdefault(metric_name, set()).add(key)
+
+    missing = []
+    for metric in reg.METRICS:
+        if str(metric) not in emitted_metrics["names"]:
+            # Reported by test_every_registered_metric_is_emitted
+            continue
+        emitted_keys = emitted_keys_by_metric.get(str(metric), set())
+        for attribute in metric.attributes:
+            if attribute not in emitted_keys:
+                missing.append(f"{metric} -> {attribute}")
+    assert not missing, (
+        "these metric attributes are documented but never emitted by the "
+        "test workload: " + ", ".join(sorted(missing))
+    )
+
+
+def test_prefix_span_lookup():
+    "prefix=True matching, which core does not use but plugin registries can."
+    hook = reg.SpanName("myplugin.hook.", "A hypothetical span family", prefix=True)
+    spans = reg.SPANS + (hook,)
+    assert reg.span_for("myplugin.hook.render_cell", spans=spans) is hook
+    assert reg.span_for("myplugin.hook.anything", spans=spans) is hook
+    assert reg.span_for("myplugin.hookish", spans=spans) is None
+    assert reg.span_for("db.query", spans=spans) is reg.DB_QUERY
+
+
+def test_exact_match_wins_over_prefix():
+    family = reg.SpanName("db.", "Greedy prefix", prefix=True)
+    spans = (family,) + reg.SPANS
+    assert reg.span_for("db.query", spans=spans) is reg.DB_QUERY
+    assert reg.span_for("db.anything-else", spans=spans) is family
+
+
+def test_attribute_values_enum_enforced():
+    outcome = reg.Attribute("myplugin.outcome", "Enum.", values={"ok", "error"})
+    open_attr = reg.Attribute("myplugin.note", "Open value set.")
+    span = reg.SpanName("myplugin.job", "Test span", (outcome, open_attr))
+    assert reg.attribute_value_allowed(span, "myplugin.outcome", "ok")
+    assert not reg.attribute_value_allowed(span, "myplugin.outcome", "surprise")
+    assert reg.attribute_value_allowed(span, "myplugin.note", "anything at all")
+    assert not reg.attribute_value_allowed(span, "not.registered", "x")
+    assert not reg.attribute_value_allowed(None, "myplugin.outcome", "ok")

tests/test_telemetry_testing_kit.py

@@ -0,0 +1,320 @@
+"""
+Tests for datasette.telemetry_testing and the public registry classes, using
+a toy plugin registry and instrumentation scope.
+"""
+
+import pytest
+
+pytest.importorskip("opentelemetry.sdk")
+
+from opentelemetry import trace as otel_trace
+
+from datasette import telemetry_registry as reg
+from datasette.telemetry import linked_root_span_kwargs
+from datasette.telemetry_testing import (
+    assert_package_never_imports_sdk,
+    assert_spans_conform,
+    assert_spans_covered,
+)
+
+SCOPE = "toyplugin"
+
+OUTCOME = reg.Attribute(
+    "toyplugin.outcome", "How the job ended.", values={"ok", "error"}
+)
+JOB_NAME = reg.Attribute("toyplugin.job", "The job's registered name.")
+JOB = reg.SpanName("toyplugin.job.run", "One job execution.", (OUTCOME, JOB_NAME))
+CHAT = reg.SpanName(
+    "toyplugin.chat ", "One model call, named `toyplugin.chat {model}`.", prefix=True
+)
+TOY_SPANS = (JOB, CHAT)
+
+toy_tracer = otel_trace.get_tracer(SCOPE, "0.1")
+
+
+def _toy_spans(otel_spans):
+    return [
+        span
+        for span in otel_spans.get_finished_spans()
+        if span.instrumentation_scope and span.instrumentation_scope.name == SCOPE
+    ]
+
+
+def _run_workload():
+    with toy_tracer.start_as_current_span(JOB) as span:
+        span.set_attribute(OUTCOME, "ok")
+        span.set_attribute(JOB_NAME, "nightly")
+    with toy_tracer.start_as_current_span("toyplugin.chat gpt-5"):
+        pass
+
+
+def test_conformance_passes_for_a_conforming_workload(otel_spans):
+    _run_workload()
+    finished = otel_spans.get_finished_spans()
+    assert_spans_conform(TOY_SPANS, finished, scope_name=SCOPE)
+    # The chat span matches the CHAT prefix entry:
+    assert_spans_covered(TOY_SPANS, finished, scope_name=SCOPE)
+
+
+def test_conformance_catches_an_unregistered_span(otel_spans):
+    with toy_tracer.start_as_current_span("toyplugin.surprise"):
+        pass
+    with pytest.raises(AssertionError, match="unregistered span"):
+        assert_spans_conform(
+            TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE
+        )
+
+
+def test_conformance_catches_an_unregistered_attribute(otel_spans):
+    with toy_tracer.start_as_current_span(JOB) as span:
+        span.set_attribute("toyplugin.stealth", 1)
+    with pytest.raises(AssertionError, match="unregistered attribute"):
+        assert_spans_conform(
+            TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE
+        )
+
+
+def test_conformance_enforces_declared_enums(otel_spans):
+    with toy_tracer.start_as_current_span(JOB) as span:
+        span.set_attribute(OUTCOME, "surprise")
+    with pytest.raises(AssertionError, match="not in the declared enum"):
+        assert_spans_conform(
+            TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE
+        )
+
+
+def test_coverage_catches_a_never_emitted_span(otel_spans):
+    with toy_tracer.start_as_current_span(JOB) as span:
+        span.set_attribute(OUTCOME, "ok")
+        span.set_attribute(JOB_NAME, "nightly")
+    # CHAT never emitted
+    with pytest.raises(AssertionError, match="never emitted"):
+        assert_spans_covered(
+            TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE
+        )
+
+
+def test_scope_filter_ignores_other_scopes(otel_spans):
+    # Spans from other scopes, including Datasette's own, are ignored:
+    other = otel_trace.get_tracer("someone-else", "1.0")
+    with other.start_as_current_span("not.in.the.toy.registry"):
+        pass
+    _run_workload()
+    assert_spans_conform(TOY_SPANS, otel_spans.get_finished_spans(), scope_name=SCOPE)
+
+
+def test_linked_root_span_kwargs_links_without_parenting(otel_spans):
+    with toy_tracer.start_as_current_span("toyplugin.cause") as cause:
+        cause_context = cause.get_span_context()
+        kwargs = linked_root_span_kwargs()
+    with toy_tracer.start_as_current_span("toyplugin.effect", **kwargs):
+        pass
+    effect = next(
+        span for span in _toy_spans(otel_spans) if span.name == "toyplugin.effect"
+    )
+    assert effect.parent is None, "must be a root, not a child"
+    assert effect.context.trace_id != cause_context.trace_id
+    assert len(effect.links) == 1
+    assert effect.links[0].context.span_id == cause_context.span_id
+
+
+def test_linked_root_span_kwargs_with_no_current_span(otel_spans):
+    kwargs = linked_root_span_kwargs()
+    assert kwargs["links"] == []
+    with toy_tracer.start_as_current_span("toyplugin.orphanless", **kwargs):
+        pass
+    span = _toy_spans(otel_spans)[0]
+    assert span.parent is None
+    assert span.links == ()
+
+
+def test_kit_module_itself_never_imports_the_sdk():
+    """
+    The kit imports the SDK lazily, so plugins can import it at module level.
+
+    conftest.py runs this test first by name. Update it there if you rename it.
+    """
+    assert_package_never_imports_sdk("datasette.telemetry_testing")
+
+
+# --- Metric conformance helpers --------------------------------------------
+
+import itertools
+
+from opentelemetry import metrics as otel_metrics_api
+
+from datasette.telemetry_testing import (
+    assert_metrics_conform,
+    assert_metrics_covered,
+)
+
+toy_meter = otel_metrics_api.get_meter(SCOPE, "0.1")
+
+# Gives each test a unique instrument name:
+_metric_ids = itertools.count()
+
+
+def _toy_metric_registry(name, kind="Counter", unit="{job}", attributes=None):
+    return (
+        reg.MetricName(
+            name,
+            kind,
+            unit,
+            "A toy metric.",
+            attributes if attributes is not None else (OUTCOME,),
+        ),
+    )
+
+
+def test_metrics_conform_passes_and_covers(otel_metrics):
+    name = f"toyplugin.jobs.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name)
+    counter = toy_meter.create_counter(name, unit="{job}", description="Jobs run")
+    counter.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+    assert_metrics_covered(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_conform_catches_unregistered_metric(otel_metrics):
+    name = f"toyplugin.stealth.{next(_metric_ids)}"
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1)
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="unregistered metric"):
+        assert_metrics_conform((), otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_conform_catches_kind_mismatch(otel_metrics):
+    name = f"toyplugin.kindclash.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name, kind="Histogram", unit="{job}")
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="registry declares Histogram"):
+        assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_conform_catches_unit_mismatch(otel_metrics):
+    name = f"toyplugin.unitclash.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name, unit="s")
+    counter = toy_meter.create_counter(name, unit="ms")
+    counter.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="unit"):
+        assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_conform_catches_unregistered_attribute(otel_metrics):
+    name = f"toyplugin.attrclash.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name)
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {"toyplugin.stealth": "x"})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="unregistered attribute"):
+        assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_conform_enforces_declared_enums(otel_metrics):
+    name = f"toyplugin.enumclash.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name)
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "surprise"})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="not in the declared enum"):
+        assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_covered_catches_never_collected(otel_metrics):
+    registered_but_never_created = _toy_metric_registry(
+        f"toyplugin.ghost.{next(_metric_ids)}"
+    )
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="never collected"):
+        assert_metrics_covered(
+            registered_but_never_created, otel_metrics, scope_name=SCOPE
+        )
+
+
+def test_metrics_covered_skips_optional_attributes(otel_metrics):
+    name = f"toyplugin.optattr.{next(_metric_ids)}"
+    error_type = reg.Attribute("toyplugin.error", "Only on failure.", optional=True)
+    registry = _toy_metric_registry(name, attributes=(OUTCOME, error_type))
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "ok"})  # No error attribute
+    otel_metrics.collect()
+    assert_metrics_covered(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_metrics_scope_filter_ignores_other_scopes(otel_metrics):
+    # Metrics from other scopes, including Datasette's own, are ignored:
+    name = f"toyplugin.scoped.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name)
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "ok"})
+    other_meter = otel_metrics_api.get_meter("someone-else-metrics", "1.0")
+    stranger = other_meter.create_counter(f"stranger.{next(_metric_ids)}", unit="x")
+    stranger.add(1)
+    otel_metrics.collect()
+    assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+# --- UpDownCounter kind + privacy walk --------------------------------------
+
+from datasette.telemetry_testing import assert_no_forbidden_values
+
+
+def test_updown_counter_kind_passes(otel_metrics):
+    name = f"toyplugin.active.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name, kind=reg.UPDOWN_COUNTER, unit="{turn}")
+    updown = toy_meter.create_up_down_counter(name, unit="{turn}")
+    updown.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_counter_registered_as_updown_fails_on_monotonicity(otel_metrics):
+    name = f"toyplugin.monoclash.{next(_metric_ids)}"
+    registry = _toy_metric_registry(name, kind=reg.UPDOWN_COUNTER, unit="{job}")
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="is_monotonic"):
+        assert_metrics_conform(registry, otel_metrics, scope_name=SCOPE)
+
+
+def test_forbidden_values_walk_catches_a_leak(otel_spans, otel_metrics):
+    secret = "sentinel-token-xyzzy"
+    with toy_tracer.start_as_current_span(JOB) as span:
+        span.set_attribute(OUTCOME, "ok")
+        span.set_attribute(JOB_NAME, f"job for {secret}")
+    with pytest.raises(AssertionError, match="sentinel-token-xyzzy"):
+        assert_no_forbidden_values(
+            {secret},
+            finished_spans=otel_spans.get_finished_spans(),
+            scope_name=SCOPE,
+        )
+
+
+def test_forbidden_values_walk_passes_a_clean_workload(otel_spans, otel_metrics):
+    _run_workload()
+    name = f"toyplugin.clean.{next(_metric_ids)}"
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {OUTCOME: "ok"})
+    otel_metrics.collect()
+    assert_no_forbidden_values(
+        {"sentinel-token-xyzzy", "alice@example.com", ""},
+        finished_spans=otel_spans.get_finished_spans(),
+        collector=otel_metrics,
+        scope_name=SCOPE,
+    )
+
+
+def test_forbidden_values_walk_checks_metric_attributes(otel_metrics):
+    secret = "leaky-metric-value"
+    name = f"toyplugin.leak.{next(_metric_ids)}"
+    counter = toy_meter.create_counter(name, unit="{job}")
+    counter.add(1, {"toyplugin.note": secret})
+    otel_metrics.collect()
+    with pytest.raises(AssertionError, match="leaky-metric-value"):
+        assert_no_forbidden_values({secret}, collector=otel_metrics, scope_name=SCOPE)

tests/test_template_context.py

@@ -10,7 +10,7 @@ from dataclasses import dataclass, field
 
 import pytest
 
-from datasette.app import Datasette, TEMPLATE_BASE_CONTEXT
+from datasette.app import TEMPLATE_BASE_CONTEXT, Datasette
 from datasette.extras import ExtraScope
 from datasette.fixtures import write_fixture_database
 from datasette.template_contexts import PAGES, documented_context_keys
@@ -40,17 +40,17 @@ def test_documented_fields():
 @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__)
 def test_context_class_fields_all_have_help(klass):
     for context_field in klass.documented_fields():
-        assert context_field.help, "{}.{} is missing documentation".format(
-            klass.__name__, context_field.name
-        )
+        assert (
+            context_field.help
+        ), f"{klass.__name__}.{context_field.name} is missing documentation"
 
 
 @pytest.mark.parametrize("klass", PAGES.values(), ids=lambda klass: klass.__name__)
 def test_context_class_has_docstring_and_documented_template(klass):
-    assert klass.__doc__, "{} is missing a docstring".format(klass.__name__)
-    assert klass.documented_template, "{} is missing a documented_template".format(
-        klass.__name__
-    )
+    assert klass.__doc__, f"{klass.__name__} is missing a docstring"
+    assert (
+        klass.documented_template
+    ), f"{klass.__name__} is missing a documented_template"
 
 
 def test_from_extra_documentation_comes_from_the_extra_class():
@@ -105,7 +105,7 @@ def isolate_extra_template_vars_plugins():
     # for the rest of the process. The contract documents plugin-free
     # Datasette core, so unregister any non-default plugin that adds
     # template variables via the extra_template_vars hook
-    from datasette.plugins import pm, DEFAULT_PLUGINS
+    from datasette.plugins import DEFAULT_PLUGINS, pm
 
     hook_plugins = {impl.plugin for impl in pm.hook.extra_template_vars.get_hookimpls()}
     removed = []
@@ -182,18 +182,18 @@ async def test_template_context_matches_documented_contract(
     undocumented = actual - documented
     no_longer_present = documented - actual
     assert not undocumented, (
-        "Undocumented keys in {} template context: {} - add them to the "
-        "page's Context class".format(page_name, sorted(undocumented))
+        f"Undocumented keys in {page_name} template context: {sorted(undocumented)} - add them to the "
+        "page's Context class"
     )
     assert not no_longer_present, (
-        "Documented keys missing from {} template context: {} - this would "
-        "break custom templates".format(page_name, sorted(no_longer_present))
+        f"Documented keys missing from {page_name} template context: {sorted(no_longer_present)} - this would "
+        "break custom templates"
     )
 
 
 def test_base_context_keys_all_have_docs():
     for name, doc in TEMPLATE_BASE_CONTEXT.items():
-        assert doc, "Base context key {} is missing docs".format(name)
+        assert doc, f"Base context key {name} is missing docs"
 
 
 def test_template_context_docs_cover_every_documented_key():
@@ -201,15 +201,14 @@ def test_template_context_docs_cover_every_documented_key():
     assert docs_path.exists(), "docs/template_context.rst is missing"
     docs = docs_path.read_text()
     for name in TEMPLATE_BASE_CONTEXT:
-        assert "``{}``".format(name) in docs, name
+        assert f"``{name}``" in docs, name
     for page_name, klass in PAGES.items():
         title = "{} page".format(klass.__name__.removesuffix("Context"))
         assert title in docs, title
         for context_field in klass.documented_fields():
-            assert "``{}``".format(context_field.name) in docs, "{} ({} page)".format(
-                context_field.name, page_name
-            )
             assert (
-                "``{}`` - ``{}``".format(context_field.name, context_field.type_name)
-                in docs
-            ), "{} type ({} page)".format(context_field.name, page_name)
+                f"``{context_field.name}``" in docs
+            ), f"{context_field.name} ({page_name} page)"
+            assert (
+                f"``{context_field.name}`` - ``{context_field.type_name}``" in docs
+            ), f"{context_field.name} type ({page_name} page)"

tests/test_token_handler.py

@@ -2,16 +2,17 @@
 Tests for the register_token_handler plugin hook.
 """
 
+import pytest
+
 from datasette.app import Datasette
 from datasette.hookspecs import hookimpl
 from datasette.plugins import pm
 from datasette.tokens import (
+    SignedTokenHandler,
     TokenHandler,
     TokenInvalid,
     TokenRestrictions,
-    SignedTokenHandler,
 )
-import pytest
 
 
 @pytest.fixture

tests/test_tracer.py

@@ -1,4 +1,5 @@
 import pytest
+
 from .fixtures import make_app_client
 
 
@@ -75,10 +76,9 @@ async def test_trace_child_tasks_resets_contextvar_on_exception():
     from datasette import tracer
 
     before = tracer.trace_task_id.get()
-    with pytest.raises(ValueError):
-        with tracer.trace_child_tasks():
-            assert tracer.trace_task_id.get() is not None
-            raise ValueError("simulated error")
+    with pytest.raises(ValueError), tracer.trace_child_tasks():
+        assert tracer.trace_task_id.get() is not None
+        raise ValueError("simulated error")
     # The contextvar must be reset even though the block raised
     assert tracer.trace_task_id.get() == before
 

tests/test_utils.py

@@ -2,23 +2,26 @@
 Tests for various datasette helper functions.
 """
 
-from datasette.app import Datasette
-from datasette import utils
-from datasette.utils.asgi import Request
-from datasette.utils.sqlite import (
-    sqlite3,
-    sqlite_hidden_table_names,
-    sqlite_table_type,
-    supports_returning,
-)
 import hashlib
 import json
 import os
 import pathlib
-import pytest
 import tempfile
 from unittest.mock import patch
 
+import pytest
+
+from datasette import utils
+from datasette.app import Datasette
+from datasette.utils.asgi import Request
+from datasette.utils.sqlite import (
+    sqlite3,
+    sqlite_derived_table_dependencies,
+    sqlite_hidden_table_names,
+    sqlite_table_type,
+    supports_returning,
+)
+
 
 @pytest.mark.parametrize(
     "path,expected",
@@ -194,7 +197,7 @@ def test_validate_sql_select_good(good_sql):
 
 @pytest.mark.parametrize("open_quote,close_quote", [('"', '"'), ("[", "]")])
 def test_detect_fts(open_quote, close_quote):
-    sql = """
+    sql = f"""
     CREATE TABLE "Dumb_Table" (
       "TreeID" INTEGER,
       "qSpecies" TEXT
@@ -209,9 +212,9 @@ def test_detect_fts(open_quote, close_quote):
       "qCaretaker" TEXT
     );
     CREATE VIEW Test_View AS SELECT * FROM Dumb_Table;
-    CREATE VIRTUAL TABLE {open}Street_Tree_List_fts{close} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open}Street_Tree_List{close});
+    CREATE VIRTUAL TABLE {open_quote}Street_Tree_List_fts{close_quote} USING FTS4 ("qAddress", "qCaretaker", "qSpecies", content={open_quote}Street_Tree_List{close_quote});
     CREATE VIRTUAL TABLE r USING rtree(a, b, c);
-    """.format(open=open_quote, close=close_quote)
+    """
     conn = utils.sqlite3.connect(":memory:")
     conn.executescript(sql)
     assert None is utils.detect_fts(conn, "Dumb_Table")
@@ -225,6 +228,8 @@ def test_detect_fts(open_quote, close_quote):
     "identifier,expected",
     (
         ("plain", "plain"),
+        ("plain\n", '"plain\n"'),
+        ("select\n", '"select\n"'),
         ("select", '"select"'),
         ("has space", '"has space"'),
         ("has'quote", '"has\'quote"'),
@@ -262,8 +267,8 @@ def test_escape_sqlite_prevents_injection():
     conn.execute("CREATE TABLE users (id INTEGER, password TEXT)")
     conn.execute("INSERT INTO users VALUES (1, 'super_secret_password')")
     malicious = "users] UNION SELECT password FROM users--"
-    conn.execute('CREATE TABLE "{}" (id INTEGER)'.format(malicious))
-    sql = "select count(*) from {}".format(utils.escape_sqlite(malicious))
+    conn.execute(f'CREATE TABLE "{malicious}" (id INTEGER)')
+    sql = f"select count(*) from {utils.escape_sqlite(malicious)}"
     results = conn.execute(sql).fetchall()
     conn.close()
     # The injected UNION must not execute - only the empty malicious table
@@ -273,16 +278,16 @@ def test_escape_sqlite_prevents_injection():
 
 @pytest.mark.parametrize("table", ("regular", "has'single quote"))
 def test_detect_fts_different_table_names(table):
-    sql = """
+    sql = f"""
     CREATE TABLE [{table}] (
       "TreeID" INTEGER,
       "qSpecies" TEXT
     );
     CREATE VIRTUAL TABLE [{table}_fts] USING FTS4 ("qSpecies", content="{table}");
-    """.format(table=table)
+    """
     conn = utils.sqlite3.connect(":memory:")
     conn.executescript(sql)
-    assert "{table}_fts".format(table=table) == utils.detect_fts(conn, table)
+    assert f"{table}_fts" == utils.detect_fts(conn, table)
     conn.close()
 
 
@@ -367,6 +372,46 @@ def test_sqlite_hidden_table_names_hides_multiline_content_fts_table():
         conn.close()
 
 
+def test_sqlite_derived_table_dependencies():
+    conn = utils.sqlite3.connect(":memory:")
+    try:
+        conn.executescript("""
+            create table docs(id integer primary key, body text);
+            create virtual table external_fts5 using fts5(
+                body, content='docs', content_rowid='id'
+            );
+            create virtual table internal_fts5 using fts5(body);
+            create virtual table contentless_fts5 using fts5(body, content='');
+            create virtual table external_fts4 using fts4(body, content="docs");
+            create virtual table internal_fts4 using fts4(body);
+            create virtual table contentless_fts4 using fts4(body, content="");
+            create table [docs, archive](body text);
+            create virtual table commented_fts5 using fts5(
+                body, tokenize='porter unicode61',
+                /* Comments and commas in quoted values must not confuse parsing. */
+                content='docs, archive'
+            );
+            create virtual table boxes using rtree(id, minx, maxx, miny, maxy);
+        """)
+
+        dependencies = sqlite_derived_table_dependencies(conn)
+
+        assert dependencies["external_fts5"] == "docs"
+        assert dependencies["external_fts4"] == "docs"
+        assert dependencies["commented_fts5"] == "docs, archive"
+        assert "contentless_fts5" not in dependencies
+        assert "contentless_fts4" not in dependencies
+        assert dependencies["internal_fts5_content"] == "internal_fts5"
+        assert dependencies["internal_fts4_content"] == "internal_fts4"
+        assert dependencies["external_fts5_data"] == "external_fts5"
+        assert dependencies["external_fts4_segments"] == "external_fts4"
+        assert dependencies["boxes_node"] == "boxes"
+        assert dependencies["boxes_parent"] == "boxes"
+        assert dependencies["boxes_rowid"] == "boxes"
+    finally:
+        conn.close()
+
+
 @pytest.mark.parametrize(
     "url,expected",
     [
@@ -690,7 +735,6 @@ def test_resolve_env_secrets(config, expected):
     [
         ({"id": "blah"}, "blah"),
         ({"id": "blah", "login": "l"}, "l"),
-        ({"id": "blah", "login": "l"}, "l"),
         ({"id": "blah", "login": "l", "username": "u"}, "u"),
         ({"login": "l", "name": "n"}, "n"),
         (

tests/test_utils_check_callable.py

@@ -1,6 +1,7 @@
-from datasette.utils.check_callable import check_callable
 import pytest
 
+from datasette.utils.check_callable import check_callable
+
 
 class AsyncClass:
     async def __call__(self):

tests/test_utils_permissions.py

@@ -1,14 +1,17 @@
+from collections.abc import Callable
+
 import pytest
+
 from datasette.app import Datasette
 from datasette.permissions import PermissionSQL
 from datasette.utils.permissions import resolve_permissions_from_catalog
-from typing import Callable, List
 
 
 @pytest.fixture
 def db():
     ds = Datasette()
     import tempfile
+
     from datasette.database import Database
 
     path = tempfile.mktemp(suffix="demo.db")
@@ -127,7 +130,7 @@ def plugin_root_deny_for_all() -> Callable[[str], PermissionSQL]:
 
 def plugin_conflicting_same_child_rules(
     user: str, parent: str, child: str
-) -> List[Callable[[str], PermissionSQL]]:
+) -> list[Callable[[str], PermissionSQL]]:
     def allow_provider(action: str) -> PermissionSQL:
         return PermissionSQL(
             """
@@ -277,9 +280,7 @@ async def test_alice_global_allow_with_specific_denies_catalog(db):
     # Alice can see everything except accounting/sales and hr/*
     assert "/accounting/sales" in res_denied(rows)
     for r in rows:
-        if r["parent"] == "hr":
-            assert r["allow"] == 0
-        elif r["resource"] == "/accounting/sales":
+        if r["parent"] == "hr" or r["resource"] == "/accounting/sales":
             assert r["allow"] == 0
         else:
             assert r["allow"] == 1

tests/test_utils_sql_analysis.py

@@ -1,7 +1,7 @@
 import pytest
 
-from datasette.utils.sqlite import sqlite3
 from datasette.utils.sql_analysis import analyze_sql_tables
+from datasette.utils.sqlite import sqlite3
 
 
 @pytest.fixture
@@ -439,7 +439,7 @@ def test_analyze_attached_database_tables(conn):
     }
 
 
-def test_analyze_clears_authorizer_on_error():
+def test_analyze_disables_authorizer_on_error():
     class FakeConnection:
         def __init__(self):
             self.authorizers = []
@@ -455,4 +455,5 @@ def test_analyze_clears_authorizer_on_error():
     with pytest.raises(sqlite3.OperationalError):
         analyze_sql_tables(conn, "bad SQL")
 
-    assert conn.authorizers[-1] is None
+    final_authorizer = conn.authorizers[-1]
+    assert final_authorizer is None or final_authorizer() == sqlite3.SQLITE_OK

tests/test_write_wrapper.py

@@ -3,14 +3,16 @@ Tests for the write_wrapper plugin hook.
 """
 
 import asyncio
+import sqlite3
+import time
 from dataclasses import dataclass
+
+import pytest
+
 from datasette.app import Datasette
 from datasette.events import Event
 from datasette.hookspecs import hookimpl
 from datasette.plugins import pm
-import pytest
-import sqlite3
-import time
 
 
 @dataclass
@@ -113,7 +115,8 @@ async def test_write_wrapper_exception_thrown_into_generator(datasette):
             def wrapper(conn):
                 try:
                     yield
-                except Exception as e:
+                except Exception as e:  # noqa: BLE001
+                    # Test helper deliberately captures whatever the wrapped write raised
                     caught["error"] = e
 
             return wrapper
@@ -232,7 +235,6 @@ async def test_write_wrapper_return_none_skips(datasette):
         @hookimpl
         def write_wrapper(datasette, database, request, transaction):
             log.append("hook-called")
-            return None
 
     pm.register(Plugin(), name="test_skip")
     try:
@@ -339,7 +341,7 @@ async def test_write_wrapper_via_api(tmp_path):
             "/test/api_test/-/insert",
             json={"row": {"name": "test"}, "return": True},
             headers={
-                "Authorization": "Bearer {}".format(token),
+                "Authorization": f"Bearer {token}",
                 "Content-Type": "application/json",
             },
         )
@@ -349,6 +351,73 @@ async def test_write_wrapper_via_api(tmp_path):
         pm.unregister(name="test_api")
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("num_sql_threads", (0, 1))
+@pytest.mark.parametrize("in_memory", (True, False))
+@pytest.mark.parametrize(
+    "operations",
+    (
+        [{"op": "add_column", "args": {"name": "extra", "type": "text"}}],
+        [{"op": "rename_column", "args": {"name": "id", "to": "renamed_id"}}],
+        [{"op": "rename_table", "args": {"to": "renamed_t"}}],
+        [
+            {"op": "add_column", "args": {"name": "extra", "type": "text"}},
+            {"op": "rename_column", "args": {"name": "id", "to": "renamed_id"}},
+            {"op": "rename_table", "args": {"to": "renamed_t"}},
+        ],
+    ),
+    ids=["add-column", "transform", "rename-table", "combined"],
+)
+async def test_write_wrapper_can_reject_alter_table_after_write(
+    tmp_path, num_sql_threads, in_memory, operations
+):
+    """Raising after yield should roll back the schema change."""
+    db_path = str(tmp_path / "demo.db")
+    ds = Datasette(
+        [] if in_memory else [db_path],
+        config={"permissions": {"alter-table": True}},
+        settings={"num_sql_threads": num_sql_threads},
+    )
+    db = (
+        ds.add_memory_database(db_path, name="demo")
+        if in_memory
+        else ds.get_database("demo")
+    )
+    await db.execute_write("CREATE TABLE t (id)")
+    await db.execute_write("INSERT INTO t (id) VALUES (1)")
+    before = await db.execute_fn(lambda conn: list(conn.iterdump()))
+
+    class Plugin:
+        __name__ = "Plugin"
+
+        @staticmethod
+        @hookimpl
+        def write_wrapper(database):
+            def wrapper(conn):
+                yield
+                raise ValueError("Rejected after write")
+
+            return wrapper if database == "demo" else None
+
+    pm.register(Plugin(), name="test_reject_alter_table")
+    try:
+        response = await ds.client.post(
+            "/demo/t/-/alter",
+            json={"operations": operations},
+        )
+        assert response.status_code == 400
+        assert response.json()["errors"] == ["Rejected after write"]
+        assert await db.execute_fn(lambda conn: list(conn.iterdump())) == before
+        assert not [
+            event
+            for event in getattr(ds, "_tracked_events", [])
+            if event.name in ("alter-table", "rename-table")
+        ]
+    finally:
+        pm.unregister(name="test_reject_alter_table")
+        ds.close()
+
+
 @pytest.mark.asyncio
 async def test_write_wrapper_change_group_pattern(datasette):
     """Test the motivating use case: activating a change group around a write."""
@@ -466,7 +535,7 @@ async def test_write_wrapper_set_authorizer(datasette, actor, table, should_deny
     try:
         request = FakeRequest(actor)
         if should_deny:
-            with pytest.raises(Exception):
+            with pytest.raises(sqlite3.DatabaseError, match="not authorized"):
                 await db.execute_write_fn(
                     lambda conn: conn.execute(
                         f"insert into {table} (value) values ('test')"