Migrate from httpx to httpx2, closes #2879

https://claude.ai/code/session_01Xdqoneq8ddvruVZETo6rFf

.github/workflows/deploy-latest.yml

@@ -14,24 +14,46 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
+    - name: Check deployment prerequisites
+      id: deployment-prerequisites
+      env:
+        GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+        LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
+      run: |
+        missing=()
+        for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do
+          if [[ -z "${!variable:-}" ]]; then
+            missing+=("$variable")
+          fi
+        done
+        if (( ${#missing[@]} )); then
+          echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}"
+          echo "available=false" >> "$GITHUB_OUTPUT"
+        else
+          echo "available=true" >> "$GITHUB_OUTPUT"
+        fi
     - name: Check out datasette
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/checkout@v7
     - name: Set up Python
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/setup-python@v6
       with:
         python-version: "3.13"
         cache: pip
     - name: Install Python dependencies
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         python -m pip install --upgrade pip
         python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1
+        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
     - name: Run tests
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
     - name: Build fixtures.db and other files needed to deploy the demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |-
         python tests/fixtures.py \
           fixtures.db \
@@ -40,13 +62,14 @@ jobs:
           plugins \
           --extra-db-filename extra_database.db
     - name: Build docs.db
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
     - name: Set up the alternate-route demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         echo '
         from datasette import hookimpl
@@ -58,6 +81,7 @@ jobs:
         ' > plugins/alternative_route.py
         cp fixtures.db fixtures2.db
     - name: And the counters writable stored query demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         cat > plugins/counters.py <<EOF
         from datasette import hookimpl
@@ -97,12 +121,15 @@ jobs:
     #     cat metadata.json
     - id: auth
       name: Authenticate to Google Cloud
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/auth@v3
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}
     - name: Set up Cloud SDK
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/setup-gcloud@v3
     - name: Deploy to Cloud Run
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       env:
         LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
       run: |-
@@ -117,16 +144,16 @@ jobs:
             --plugins-dir=plugins \
             --branch=$GITHUB_SHA \
             --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1 --crossdb --root" \
+            --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
             --install 'datasette-ephemeral-tables>=0.2.2' \
             --service "datasette-latest$SUFFIX" \
             --secret $LATEST_DATASETTE_SECRET
-    - name: Deploy to docs as well (only for main)
-      if: ${{ github.ref == 'refs/heads/main' }}
+    - name: Upload latest documentation database to S3 (only for main)
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
       run: |-
-        # Deploy docs.db to a different service
-        datasette publish cloudrun docs.db \
-            --branch=$GITHUB_SHA \
-            --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1" \
-            --service=datasette-docs-latest
+        # Keep development documentation separate from the stable release database.
+        s3-credentials put-object datasette-docs latest/docs.db docs.db \
+          --content-type application/octet-stream

.github/workflows/publish.yml

@@ -2,7 +2,7 @@ name: Publish Python Package
 
 on:
   release:
-    types: [created]
+    types: [published]
 
 permissions:
   contents: read
@@ -51,6 +51,8 @@ jobs:
     - name: Publish
       uses: pypa/gh-action-pypi-publish@release/v1
 
+  # After the first non-prerelease 1.0 release, disable this job on 0.65.x,
+  # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs.
   deploy_static_docs:
     runs-on: ubuntu-latest
     needs: [deploy]
@@ -66,26 +68,20 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1
+        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
     - name: Build docs.db
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
-    - id: auth
-      name: Authenticate to Google Cloud
-      uses: google-github-actions/auth@v2
-      with:
-        credentials_json: ${{ secrets.GCP_SA_KEY }}
-    - name: Set up Cloud SDK
-      uses: google-github-actions/setup-gcloud@v3
-    - name: Deploy stable-docs.datasette.io to Cloud Run
+    - name: Upload stable documentation database to S3
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
       run: |-
-        gcloud config set run/region us-central1
-        gcloud config set project datasette-222320
-        datasette publish cloudrun docs.db \
-            --service=datasette-docs-stable
+        s3-credentials put-object datasette-docs docs.db docs.db \
+          --content-type application/octet-stream
 
   deploy_docker:
     runs-on: ubuntu-latest

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.0-slim-bullseye as build
+FROM python:3.11-slim-bookworm AS build
 
 # Version of Datasette to install, e.g. 0.55
 #   docker build . -t datasette --build-arg VERSION=0.55

datasette/app.py

@@ -28,7 +28,7 @@ import urllib.parse
 from concurrent import futures
 from pathlib import Path
 
-import httpx
+import httpx2
 from itsdangerous import BadSignature, URLSafeSerializer
 from jinja2 import (
     ChoiceLoader,
@@ -49,14 +49,8 @@ from .events import Event
 from .plugins import DEFAULT_PLUGINS, get_plugins, pm
 from .renderer import json_renderer
 from .resources import DatabaseResource, TableResource
-from .telemetry import (
-    TelemetryMiddleware,
-    clamp_http_method,
-    request_span,
-    tracer,
-)
-from .telemetry_registry import HTTP_ROUTE, STARTUP
 from .tokens import TokenInvalid
+from .tracer import AsgiTracer
 from .url_builder import Urls
 from .utils import (
     SPATIALITE_FUNCTIONS,
@@ -293,6 +287,11 @@ SETTINGS = (
         False,
         "Allow display of template debug information with ?_context=1",
     ),
+    Setting(
+        "trace_debug",
+        False,
+        "Allow display of SQL trace debug information with ?_trace=1",
+    ),
     Setting("base_url", "/", "Datasette URLs should use this base path"),
 )
 _HASH_URLS_REMOVED = "The hash_urls setting has been removed, try the datasette-hashed-urls plugin instead"
@@ -316,7 +315,7 @@ def _permission_cache_key(actor, action, parent, child):
     actor_key = (
         json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None
     )
-    return (actor_key, action, parent, child)
+    return (actor_key, action.name, parent, action.normalize_child(child))
 
 
 async def favicon(request, send):
@@ -779,73 +778,57 @@ class Datasette:
         # This must be called for Datasette to be in a usable state
         if self._startup_invoked:
             return
-        # `datasette serve` calls invoke_startup() before uvicorn starts, so
-        # on the CLI path every span its children create - the register_*
-        # hook dispatches, the internal catalog's db.query/db.write spans,
-        # and the prepare_connection warm-up of the read connections those
-        # touch - would otherwise be its own orphan root trace: around twenty
-        # of them on a fresh instance. Bracketing the whole thing gives them
-        # somewhere to belong. An ASGI-hosted or programmatic deployment
-        # reaches here instead through AsgiRunOnFirstRequest, in which case
-        # this span nests under the first request's own span - honest enough,
-        # since it genuinely is that request's latency.
-        # A connection warmed lazily later, by a request touching a new
-        # database for the first time, nests under that request instead:
-        # this span has already ended by then.
-        with tracer.start_as_current_span(STARTUP):
-            # Register event classes
-            event_classes = []
-            for hook in pm.hook.register_events(datasette=self):
-                extra_classes = await await_me_maybe(hook)
-                if extra_classes:
-                    event_classes.extend(extra_classes)
-            self.event_classes = tuple(event_classes)
+        # Register event classes
+        event_classes = []
+        for hook in pm.hook.register_events(datasette=self):
+            extra_classes = await await_me_maybe(hook)
+            if extra_classes:
+                event_classes.extend(extra_classes)
+        self.event_classes = tuple(event_classes)
 
-            # Register actions, but watch out for duplicate name/abbr
-            action_names = {}
-            action_abbrs = {}
-            for hook in pm.hook.register_actions(datasette=self):
-                if hook:
-                    for action in hook:
-                        if (
-                            action.name in action_names
-                            and action != action_names[action.name]
-                        ):
-                            raise StartupError(f"Duplicate action name: {action.name}")
-                        if (
-                            action.abbr
-                            and action.abbr in action_abbrs
-                            and action != action_abbrs[action.abbr]
-                        ):
-                            raise StartupError(f"Duplicate action abbr: {action.abbr}")
-                        action_names[action.name] = action
-                        if action.abbr:
-                            action_abbrs[action.abbr] = action
-                        self.actions[action.name] = action
+        # Register actions, but watch out for duplicate name/abbr
+        action_names = {}
+        action_abbrs = {}
+        for hook in pm.hook.register_actions(datasette=self):
+            if hook:
+                for action in hook:
+                    if (
+                        action.name in action_names
+                        and action != action_names[action.name]
+                    ):
+                        raise StartupError(f"Duplicate action name: {action.name}")
+                    if (
+                        action.abbr
+                        and action.abbr in action_abbrs
+                        and action != action_abbrs[action.abbr]
+                    ):
+                        raise StartupError(f"Duplicate action abbr: {action.abbr}")
+                    action_names[action.name] = action
+                    if action.abbr:
+                        action_abbrs[action.abbr] = action
+                    self.actions[action.name] = action
 
-            # Register column types (classes, not instances)
-            self._column_types = {}
-            for hook in pm.hook.register_column_types(datasette=self):
-                if hook:
-                    for ct_cls in hook:
-                        if ct_cls.name in self._column_types:
-                            raise StartupError(
-                                f"Duplicate column type name: {ct_cls.name}"
-                            )
-                        self._column_types[ct_cls.name] = ct_cls
+        # Register column types (classes, not instances)
+        self._column_types = {}
+        for hook in pm.hook.register_column_types(datasette=self):
+            if hook:
+                for ct_cls in hook:
+                    if ct_cls.name in self._column_types:
+                        raise StartupError(f"Duplicate column type name: {ct_cls.name}")
+                    self._column_types[ct_cls.name] = ct_cls
 
-            for hook in pm.hook.prepare_jinja2_environment(
-                env=self._jinja_env, datasette=self
-            ):
-                await await_me_maybe(hook)
-            # Ensure internal tables and metadata are populated before startup hooks
-            await self._refresh_schemas()
-            await self._save_queries_from_config()
-            # Load column_types from config into internal DB
-            await self._apply_column_types_config()
-            for hook in pm.hook.startup(datasette=self):
-                await await_me_maybe(hook)
-            self._startup_invoked = True
+        for hook in pm.hook.prepare_jinja2_environment(
+            env=self._jinja_env, datasette=self
+        ):
+            await await_me_maybe(hook)
+        # Ensure internal tables and metadata are populated before startup hooks
+        await self._refresh_schemas()
+        await self._save_queries_from_config()
+        # Load column_types from config into internal DB
+        await self._apply_column_types_config()
+        for hook in pm.hook.startup(datasette=self):
+            await await_me_maybe(hook)
+        self._startup_invoked = True
 
     def sign(self, value, namespace="default"):
         return URLSafeSerializer(self._secret, namespace).dumps(value)
@@ -1549,15 +1532,28 @@ class Datasette:
         conn.row_factory = sqlite3.Row
         conn.text_factory = lambda x: str(x, "utf-8", "replace")
         if self.sqlite_extensions and database != INTERNAL_DB_NAME:
+            # Extension loading is only enabled for as long as it takes to
+            # load the configured extensions. Leaving it enabled would let
+            # anyone who can execute SQL call load_extension() themselves.
             conn.enable_load_extension(True)
-            for extension in self.sqlite_extensions:
-                # "extension" is either a string path to the extension
-                # or a 2-item tuple that specifies which entrypoint to load.
-                if isinstance(extension, tuple):
-                    path, entrypoint = extension
-                    conn.execute("SELECT load_extension(?, ?)", [path, entrypoint])
-                else:
-                    conn.execute("SELECT load_extension(?)", [extension])
+            try:
+                for extension in self.sqlite_extensions:
+                    # "extension" is either a string path to the extension
+                    # or a 2-item tuple that specifies which entrypoint to load.
+                    if isinstance(extension, tuple):
+                        path, entrypoint = extension
+                        if sys.version_info >= (3, 12):
+                            conn.load_extension(path, entrypoint=entrypoint)
+                        else:
+                            # Connection.load_extension() only gained the
+                            # entrypoint argument in Python 3.12
+                            conn.execute(
+                                "SELECT load_extension(?, ?)", [path, entrypoint]
+                            )
+                    else:
+                        conn.load_extension(extension)
+            finally:
+                conn.enable_load_extension(False)
         if self.setting("cache_size_kb"):
             conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}")
         # pylint: disable=no-member
@@ -1750,8 +1746,145 @@ class Datasette:
         sql, params = await build_allowed_resources_sql(
             self, actor, action, parent=parent, include_is_private=include_is_private
         )
+        if action == "view-table":
+            sql, params = await self._apply_derived_table_permissions_to_sql(
+                sql,
+                params,
+                actor=actor,
+                parent=parent,
+                include_is_private=include_is_private,
+            )
         return ResourcesSQL(sql, params)
 
+    async def _allowed_derived_table_source(
+        self, database, source, *, actor, dependencies
+    ):
+        """Check an immediate source, denying sources that are themselves derived."""
+        if any(
+            TableResource.normalize_child(table)
+            == TableResource.normalize_child(source)
+            for table in dependencies
+        ):
+            return False
+        # The source has no dependency in this map. Evaluate its own permission
+        # and prerequisites without starting another dependency check.
+        verdicts = await self._allowed_many(
+            actions=["view-table"],
+            resource=TableResource(database, source),
+            actor=actor,
+            check_derived=False,
+        )
+        return verdicts["view-table"]
+
+    async def _apply_derived_table_permissions_to_sql(
+        self,
+        sql,
+        params,
+        *,
+        actor,
+        parent,
+        include_is_private,
+    ):
+        databases = (
+            [(parent, self.databases[parent])]
+            if parent in self.databases
+            else ([] if parent is not None else list(self.databases.items()))
+        )
+        dependency_maps = dict(
+            zip(
+                (name for name, _ in databases),
+                await asyncio.gather(
+                    *(db.derived_table_dependencies() for _, db in databases)
+                ),
+            )
+        )
+        dependencies = [
+            (database_name, child, source)
+            for database_name, dependency_map in dependency_maps.items()
+            for child, source in dependency_map.items()
+        ]
+        if not dependencies:
+            return sql, params
+
+        sources = sorted(
+            {(database_name, source) for database_name, _, source in dependencies}
+        )
+        actor_verdicts = await asyncio.gather(
+            *(
+                self._allowed_derived_table_source(
+                    database_name,
+                    source,
+                    actor=actor,
+                    dependencies=dependency_maps[database_name],
+                )
+                for database_name, source in sources
+            )
+        )
+        actor_allowed = dict(zip(sources, actor_verdicts))
+
+        anonymous_allowed = {}
+        if include_is_private:
+            anonymous_verdicts = await asyncio.gather(
+                *(
+                    self._allowed_derived_table_source(
+                        database_name,
+                        source,
+                        actor=None,
+                        dependencies=dependency_maps[database_name],
+                    )
+                    for database_name, source in sources
+                )
+            )
+            anonymous_allowed = dict(zip(sources, anonymous_verdicts))
+
+        wrapped_params = dict(params)
+        derived_rows = [
+            [
+                database_name,
+                child,
+                int(actor_allowed[(database_name, source)]),
+                *(
+                    [int(anonymous_allowed[(database_name, source)])]
+                    if include_is_private
+                    else []
+                ),
+            ]
+            for database_name, child, source in dependencies
+        ]
+        derived_param = "_datasette_derived_permissions"
+        while derived_param in wrapped_params:
+            derived_param += "_"
+        wrapped_params[derived_param] = json.dumps(derived_rows)
+
+        derived_columns = "parent, child, source_allowed"
+        select_columns = "allowed.parent, allowed.child, allowed.reason"
+        if include_is_private:
+            derived_columns += ", source_anonymous_allowed"
+            select_columns += (
+                ", CASE WHEN derived.source_anonymous_allowed = 0 "
+                "THEN 1 ELSE allowed.is_private END AS is_private"
+            )
+        wrapped_sql = f"""
+WITH derived_permissions({derived_columns}) AS (
+  SELECT
+    json_extract(value, '$[0]'),
+    json_extract(value, '$[1]'),
+    json_extract(value, '$[2]')
+    {", json_extract(value, '$[3]')" if include_is_private else ""}
+  FROM json_each(:{derived_param})
+),
+allowed AS (
+{sql}
+)
+SELECT {select_columns}
+FROM allowed
+LEFT JOIN derived_permissions AS derived
+  ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE
+WHERE COALESCE(derived.source_allowed, 1) = 1
+ORDER BY allowed.parent, allowed.child
+""".strip()
+        return wrapped_sql, wrapped_params
+
     async def allowed_resources(
         self,
         action: str,
@@ -1954,6 +2087,12 @@ class Datasette:
             )
             # {"edit-schema": True, "drop-table": True, "insert-row": False}
         """
+        return await self._allowed_many(
+            actions=actions, resource=resource, actor=actor, check_derived=True
+        )
+
+    async def _allowed_many(self, *, actions, resource, actor, check_derived):
+        """Evaluate permissions, optionally applying the one-hop source policy."""
         from datasette.permissions import (
             _permission_check_cache,
             _skip_permission_checks,
@@ -1991,7 +2130,7 @@ class Datasette:
         to_check = []
         for name in expanded:
             if cache is not None:
-                key = _permission_cache_key(actor, name, parent, child)
+                key = _permission_cache_key(actor, self.actions[name], parent, child)
                 if key in cache:
                     final[name] = cache[key]
                     continue
@@ -2007,6 +2146,28 @@ class Datasette:
                 child=child,
             )
 
+        if (
+            check_derived
+            and "view-table" in to_check
+            and raw.get("view-table")
+            and isinstance(resource, TableResource)
+            and parent in self.databases
+        ):
+            dependencies = await self.databases[parent].derived_table_dependencies()
+            source = next(
+                (
+                    source
+                    for table, source in dependencies.items()
+                    if TableResource.normalize_child(table)
+                    == TableResource.normalize_child(child)
+                ),
+                None,
+            )
+            if source is not None:
+                raw["view-table"] = await self._allowed_derived_table_source(
+                    parent, source, actor=actor, dependencies=dependencies
+                )
+
         def resolve(name):
             # final verdict = own rules AND verdict of also_requires chain
             if name in final:
@@ -2024,7 +2185,9 @@ class Datasette:
         # Cache the freshly computed checks
         if cache is not None:
             for name in to_check:
-                cache[_permission_cache_key(actor, name, parent, child)] = final[name]
+                cache[
+                    _permission_cache_key(actor, self.actions[name], parent, child)
+                ] = final[name]
 
         # Log every check (including cache hits) for the debug page,
         # dependencies before the actions that required them
@@ -2466,7 +2629,7 @@ class Datasette:
     ):
         data = {"a": actor}
         if expire_after:
-            expires_at = int(time.time()) + (24 * 60 * 60)
+            expires_at = int(time.time()) + expire_after
             data["e"] = baseconv.base62.encode(expires_at)
         response.set_cookie("ds_actor", self.sign(data, "actor"))
 
@@ -2832,7 +2995,7 @@ class Datasette:
         This is the single entry point used by both AsgiLifespan (so
         real deployments finish startup before accepting requests) and
         AsgiRunOnFirstRequest (the fallback for hosts that never send
-        lifespan events, e.g. DatasetteClient's httpx.ASGITransport), and
+        lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and
         `datasette serve` (cli.py) calls it too. The fast path below checks
         both `_startup_invoked` and `_setup_db_done` - not just the former -
         so that a bare `await ds.invoke_startup()` made by a caller ahead of
@@ -2861,6 +3024,8 @@ class Datasette:
             self.close()
 
         asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self)
+        if self.setting("trace_debug"):
+            asgi = AsgiTracer(asgi)
         asgi = AsgiLifespan(
             asgi,
             on_startup=[self._startup_sequence],
@@ -2869,12 +3034,6 @@ class Datasette:
         asgi = AsgiRunOnFirstRequest(asgi, on_startup=[self._startup_sequence])
         for wrapper in pm.hook.asgi_wrapper(datasette=self):
             asgi = wrapper(asgi)
-        # Outermost, deliberately: plugin asgi_wrapper() middleware, the
-        # CSRF layer and the first-request startup fallback all run *inside*
-        # this span, so a span created by an instrumented plugin - or by
-        # startup work triggered by the first request - parents to the
-        # request instead of becoming its own orphan root trace.
-        asgi = TelemetryMiddleware(asgi)
         return asgi
 
 
@@ -2912,6 +3071,50 @@ class DatasetteRouter:
             receive,
             max_post_body_bytes=self.ds.setting("max_post_body_bytes"),
         )
+        match, view = resolve_routes(self.routes, path)
+        is_static = view is favicon or getattr(view, "_datasette_static", False)
+        original_send = send
+
+        async def send(message):
+            if message["type"] == "http.response.start" and not (
+                is_static and message["status"] in (200, 304)
+            ):
+                # Decide privacy after rendering, including for streaming responses
+                # and error handlers. A public primary resource can still include
+                # private labels, actor navigation, or cookie-dependent content.
+                headers = list(message.get("headers", []))
+                personalized = (
+                    request.actor is not None
+                    or "cookie" in request.headers
+                    or "authorization" in request.headers
+                    or any(key.lower() == b"set-cookie" for key, _ in headers)
+                )
+                if personalized:
+                    headers = [
+                        (key, value)
+                        for key, value in headers
+                        if key.lower() != b"cache-control"
+                    ]
+                    headers.append((b"cache-control", b"private, no-store"))
+
+                # Anonymous responses must not be reused for credentialed requests.
+                # Preserve any additional variation specified by views or plugins.
+                vary = [
+                    part.strip()
+                    for key, value in headers
+                    if key.lower() == b"vary"
+                    for part in value.split(b",")
+                    if part.strip()
+                ]
+                if b"*" not in vary:
+                    for name in (b"Cookie", b"Authorization"):
+                        if name.lower() not in {part.lower() for part in vary}:
+                            vary.append(name)
+                headers = [(k, v) for k, v in headers if k.lower() != b"vary"]
+                headers.append((b"vary", b", ".join(vary)))
+                message = dict(message, headers=headers)
+            await original_send(message)
+
         # Populate request_messages if ds_messages cookie is present
         try:
             request._messages = self.ds.unsign(
@@ -2951,30 +3154,11 @@ class DatasetteRouter:
             return await self.handle_401(request, send, token_error)
         scope_modifications["actor"] = actor or default_actor
         scope = dict(scope, **scope_modifications)
-
-        match, view = resolve_routes(self.routes, path)
+        request.scope = scope
 
         if match is None:
-            # No route matched, so the span keeps the bare method name it was
-            # given at the edge and gets no http.route. That is what semantic
-            # conventions ask for when the route is unknown.
             return await self.handle_404(request, send)
 
-        # The request span was started at the ASGI edge, before routing, so it
-        # carries only the method as a name. Now that the route is known, give
-        # it the `{method} {route}` shape semantic conventions want, and the
-        # http.route attribute - the low-cardinality counterpart to url.path,
-        # and so the one to group by.
-        span = request_span(scope)
-        if span is not None:
-            route = match.re.pattern
-            span.set_attribute(HTTP_ROUTE, route)
-            # Clamped, for the same reason the middleware clamps it: the method
-            # is a client-controlled string, and an unclamped one here would
-            # put attacker-supplied text back into the span name that the
-            # middleware just kept out of it.
-            span.update_name(f"{clamp_http_method(request.method)} {route}")
-
         new_scope = dict(scope, url_route={"kwargs": match.groupdict()})
         request.scope = new_scope
         try:
@@ -3285,14 +3469,14 @@ class DatasetteClient:
         with _DatasetteClientContext():
             if skip_permission_checks:
                 with SkipPermissions():
-                    async with httpx.AsyncClient(
-                        transport=httpx.ASGITransport(app=self.app),
+                    async with httpx2.AsyncClient(
+                        transport=httpx2.ASGITransport(app=self.app),
                         cookies=kwargs.pop("cookies", None),
                     ) as client:
                         return await getattr(client, method)(self._fix(path), **kwargs)
             else:
-                async with httpx.AsyncClient(
-                    transport=httpx.ASGITransport(app=self.app),
+                async with httpx2.AsyncClient(
+                    transport=httpx2.ASGITransport(app=self.app),
                     cookies=kwargs.pop("cookies", None),
                 ) as client:
                     return await getattr(client, method)(self._fix(path), **kwargs)
@@ -3339,10 +3523,10 @@ class DatasetteClient:
             method: HTTP method (e.g., "GET", "POST", "PUT")
             path: The path to request
             skip_permission_checks: If True, bypass all permission checks for this request
-            **kwargs: Additional arguments to pass to httpx
+            **kwargs: Additional arguments to pass to httpx2
 
         Returns:
-            httpx.Response: The response from the request
+            httpx2.Response: The response from the request
         """
         from datasette.permissions import SkipPermissions
 
@@ -3351,16 +3535,16 @@ class DatasetteClient:
         with _DatasetteClientContext():
             if skip_permission_checks:
                 with SkipPermissions():
-                    async with httpx.AsyncClient(
-                        transport=httpx.ASGITransport(app=self.app),
+                    async with httpx2.AsyncClient(
+                        transport=httpx2.ASGITransport(app=self.app),
                         cookies=kwargs.pop("cookies", None),
                     ) as client:
                         return await client.request(
                             method, self._fix(path, avoid_path_rewrites), **kwargs
                         )
             else:
-                async with httpx.AsyncClient(
-                    transport=httpx.ASGITransport(app=self.app),
+                async with httpx2.AsyncClient(
+                    transport=httpx2.ASGITransport(app=self.app),
                     cookies=kwargs.pop("cookies", None),
                 ) as client:
                     return await client.request(

datasette/database.py

@@ -1,45 +1,19 @@
 import asyncio
 import atexit
-import contextvars
 import inspect
 import os
 import queue
 import sys
 import tempfile
 import threading
-import time
 import uuid
 from collections import namedtuple
 from pathlib import Path
 
 import sqlite_utils
-from opentelemetry import context as otel_context_api
-from opentelemetry.trace import Link, Status, StatusCode, get_current_span
 
 from .inspect import inspect_hash
-from .telemetry import sql_attribute, sql_operation_name, tracer
-from .telemetry_registry import (
-    DB_COLLECTION_NAME,
-    DB_NAMESPACE,
-    DB_OPERATION_NAME,
-    DB_QUERY,
-    DB_QUERY_EXECUTE,
-    DB_QUERY_TEXT,
-    DB_SYSTEM,
-    DB_WRITE_EXECUTE,
-    DB_WRITE_QUEUE_WAIT,
-    EXECUTEMANY,
-    EXECUTESCRIPT,
-    INTERRUPTED,
-    ISOLATED_CONNECTION,
-    PARAM_COUNT,
-    PARAM_SETS,
-    ROWS_RETURNED,
-    SQL_ERROR_SUPPRESSED,
-    TIME_LIMIT_MS,
-    TRANSACTION,
-    TRUNCATED,
-)
+from .tracer import trace
 from .utils import (
     call_with_supported_arguments,
     detect_fts,
@@ -55,7 +29,7 @@ from .utils import (
     table_columns,
 )
 from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables
-from .utils.sqlite import sqlite_hidden_table_names
+from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names
 
 connections = threading.local()
 
@@ -111,6 +85,7 @@ class Database:
         self.cached_hash = None
         self.cached_size = None
         self._cached_table_counts = None
+        self._cached_derived_table_dependencies = None
         self._write_thread = None
         self._write_queue = None
         self._closed = False
@@ -272,26 +247,30 @@ class Database:
         return_all=False,
         returning_limit=EXECUTE_WRITE_RETURNING_LIMIT,
         transaction=True,
+        time_limit_ms=2000,
     ):
         self._check_not_closed()
         if returning_limit < 0:
             raise ValueError("returning_limit must be >= 0")
 
-        def _inner(conn):
+        def execute_sql(conn):
             cursor = conn.execute(sql, params or [])
             return ExecuteWriteResult.from_cursor(
                 cursor, return_all=return_all, returning_limit=returning_limit
             )
 
-        with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
-            span.set_attribute(DB_SYSTEM, "sqlite")
-            span.set_attribute(DB_NAMESPACE, self.name)
-            span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
-            operation_name = sql_operation_name(sql)
-            if operation_name:
-                span.set_attribute(DB_OPERATION_NAME, operation_name)
-            if params:
-                span.set_attribute(PARAM_COUNT, len(params))
+        def _inner(conn):
+            try:
+                if time_limit_ms is None:
+                    return execute_sql(conn)
+                with sqlite_timelimit(conn, time_limit_ms):
+                    return execute_sql(conn)
+            except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:
+                if e.args == ("interrupted",):
+                    raise QueryInterrupted(e, sql, params)
+                raise
+
+        with trace("sql", database=self.name, sql=sql.strip(), params=params):
             results = await self.execute_write_fn(
                 _inner, block=block, request=request, transaction=transaction
             )
@@ -303,15 +282,7 @@ class Database:
         def _inner(conn):
             return conn.executescript(sql)
 
-        # No db.operation.name here, deliberately: executescript() runs
-        # several semicolon-separated statements, and semantic conventions
-        # say the attribute should not be extracted from query text that
-        # can hold more than one operation - see sql_operation_name().
-        with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
-            span.set_attribute(DB_SYSTEM, "sqlite")
-            span.set_attribute(DB_NAMESPACE, self.name)
-            span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
-            span.set_attribute(EXECUTESCRIPT, True)
+        with trace("sql", database=self.name, sql=sql.strip(), executescript=True):
             results = await self.execute_write_fn(
                 _inner, block=block, transaction=False, request=request
             )
@@ -331,22 +302,13 @@ class Database:
 
             return conn.executemany(sql, count_params(params_seq)), count
 
-        with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
-            span.set_attribute(DB_SYSTEM, "sqlite")
-            span.set_attribute(DB_NAMESPACE, self.name)
-            span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
-            span.set_attribute(EXECUTEMANY, True)
-            # A single statement run with many parameter sets, so unlike
-            # execute_write_script() there is exactly one operation to name.
-            operation_name = sql_operation_name(sql)
-            if operation_name:
-                span.set_attribute(DB_OPERATION_NAME, operation_name)
+        with trace(
+            "sql", database=self.name, sql=sql.strip(), executemany=True
+        ) as kwargs:
             results, count = await self.execute_write_fn(
                 _inner, block=block, request=request
             )
-            # count is the number of parameter *sets* consumed by
-            # executemany(), not a row count - executemany returns no rows.
-            span.set_attribute(PARAM_SETS, count)
+            kwargs["count"] = count
         return results
 
     async def execute_isolated_fn(self, fn):
@@ -372,18 +334,9 @@ class Database:
             return _run()
         if not write:
             # Immutable database - no writes can ever occur, so there is no
-            # write queue to block; run against a fresh read-only connection.
-            # A fresh copy_context() is required per submit (not one shared
-            # copy reused across calls): concurrent execution of the same
-            # Context raises "RuntimeError: cannot enter context ... already
-            # entered". This propagates the caller's otel context (e.g. the
-            # enclosing db.query span) onto the worker thread.
-            #
-            # It also propagates every *other* ContextVar - see the note in
-            # execute_fn() for why that is safe.
-            ctx = contextvars.copy_context()
+            # write queue to block; run against a fresh read-only connection
             return await asyncio.get_running_loop().run_in_executor(
-                self.ds.executor, ctx.run, _run
+                self.ds.executor, _run
             )
         # Threaded mode - send to write thread
         return await self._send_to_write_thread(fn, isolated_connection=True)
@@ -414,6 +367,15 @@ class Database:
                     result = fn(self._write_connection)
             else:
                 result = fn(self._write_connection)
+            if not block:
+                # There is no write thread here, so the write has already
+                # finished. Hand back the same (task_id, reply_future) shape
+                # _send_to_write_thread() returns, with the future already
+                # resolved, so the block=False path below is identical in
+                # both modes.
+                reply_future = asyncio.get_running_loop().create_future()
+                reply_future.set_result(result)
+                result = (uuid.uuid4(), reply_future)
         else:
             result = await self._send_to_write_thread(
                 fn, block=block, transaction=transaction
@@ -485,27 +447,11 @@ class Database:
             )
             self._write_thread.name = f"_execute_writes for database {self.name}"
             self._write_thread.start()
-        task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
+        task_id = uuid.uuid4()
         loop = asyncio.get_running_loop()
         reply_future = loop.create_future()
-        # Captured here, on the event loop, at enqueue time: the otel
-        # Context (carrying the enclosing db.query span, if any) and the
-        # timestamp used to build the db.write.queue_wait span once this
-        # task is dequeued on the write thread. `block` travels with the
-        # task too, because it decides whether that context is this task's
-        # parent or only a link target - see `_execute_writes`.
         self._write_queue.put(
-            WriteTask(
-                fn,
-                task_id,
-                loop,
-                reply_future,
-                isolated_connection,
-                transaction,
-                otel_context_api.get_current(),
-                time.time_ns(),
-                block,
-            )
+            WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction)
         )
         if block:
             return await reply_future
@@ -519,16 +465,6 @@ class Database:
         conn = None
         try:
             conn = self.connect(write=True)
-            # This warm-up runs before any write has ever been queued, so
-            # there is no captured caller context to attach - and a raw
-            # threading.Thread does not inherit the context of whoever started
-            # it. Spans created by plugin hooks here are therefore roots even
-            # when the write thread is started from inside invoke_startup():
-            # its datasette.startup span is current on the event loop but does
-            # not cross this thread boundary. Read connections differ - they
-            # warm up inside executor tasks submitted with copy_context(), so
-            # their prepare_connection spans do nest under whoever triggered
-            # them.
             self.ds._prepare_connection(conn, self.name)
         except Exception as e:  # noqa: BLE001
             # Stored and re-raised to whoever queues the next write
@@ -543,119 +479,40 @@ class Database:
                         # Best-effort close as the write thread exits
                         pass
                 return
-            # `task.block` decides how this task's spans relate to the
-            # context captured at enqueue time:
-            #
-            # - block=True: the caller genuinely awaits the reply, so
-            #   containment is accurate. Restore that context as current
-            #   (attach below) so db.write.queue_wait/db.write.execute parent
-            #   normally to the request that queued them. The token must be
-            #   detached below in `finally` - a leaked token silently
-            #   poisons this thread's ambient context for every write
-            #   processed after it, and a *wrong*-token detach only logs a
-            #   warning rather than raising, so this pairing is load-bearing
-            #   and easy to get wrong silently.
-            # - block=False: the caller returned already without awaiting,
-            #   so the enqueueing span may already have closed (and
-            #   exported) before this task's spans even start - parenting to
-            #   it would make a child appear to outlive its already-closed
-            #   parent, which OTel allows but which renders badly in most
-            #   trace UIs. The enqueueing request *caused* this write
-            #   without *containing* it, so nothing is attached here -
-            #   instead each write span is started as its own root (explicit
-            #   empty `context=`, so the write thread's ambient context
-            #   cannot supply a parent either) carrying one `Link` back to
-            #   the enqueueing span's context, built once into
-            #   `write_span_kwargs` and spread into every start_span call
-            #   below.
-            token = None
-            write_span_kwargs = {}
-            if task.block:
-                token = otel_context_api.attach(task.otel_context)
+            exception = None
+            result = None
+            if conn_exception is not None:
+                exception = conn_exception
+            elif task.isolated_connection:
+                try:
+                    isolated_connection = self.connect(write=True)
+                    try:
+                        result = task.fn(isolated_connection)
+                    finally:
+                        isolated_connection.close()
+                        try:
+                            self._all_file_connections.remove(isolated_connection)
+                        except ValueError:
+                            # Was probably a memory connection
+                            pass
+                except Exception as e:  # noqa: BLE001
+                    # Write thread must survive any task failure or the database wedges
+                    sys.stderr.write(f"{e}\n")
+                    sys.stderr.flush()
+                    exception = e
             else:
-                enqueueing_span_context = get_current_span(
-                    task.otel_context
-                ).get_span_context()
-                # No attributes on the link: there is only one kind of link
-                # here, so naming the relationship would be a constant that
-                # carries no information a consumer does not already have
-                # from the link's existence.
-                links = (
-                    [Link(enqueueing_span_context)]
-                    if enqueueing_span_context.is_valid
-                    else []
-                )
-                write_span_kwargs = {
-                    "context": otel_context_api.Context(),
-                    "links": links,
-                }
-            try:
-                exception = None
-                result = None
-                # Explicit start_time/end_time rather than a `with` block:
-                # this span's duration is the time the task actually spent
-                # waiting in the queue (enqueue -> dequeue), not the near-
-                # zero time spent constructing/ending the span object here.
-                tracer.start_span(
-                    DB_WRITE_QUEUE_WAIT,
-                    start_time=task.enqueued_at_ns,
-                    **write_span_kwargs,
-                ).end(end_time=time.time_ns())
-                if conn_exception is not None:
-                    # fn never runs in this branch, so there is nothing to
-                    # wrap in a db.write.execute span.
-                    exception = conn_exception
-                elif task.isolated_connection:
-                    try:
-                        with tracer.start_as_current_span(
-                            DB_WRITE_EXECUTE, **write_span_kwargs
-                        ) as span:
-                            span.set_attribute(
-                                ISOLATED_CONNECTION,
-                                task.isolated_connection,
-                            )
-                            span.set_attribute(TRANSACTION, task.transaction)
-                            isolated_connection = self.connect(write=True)
-                            try:
-                                result = task.fn(isolated_connection)
-                            finally:
-                                isolated_connection.close()
-                                try:
-                                    self._all_file_connections.remove(
-                                        isolated_connection
-                                    )
-                                except ValueError:
-                                    # Was probably a memory connection
-                                    pass
-                    except Exception as e:  # noqa: BLE001
-                        # Write thread must survive any task failure or the database wedges
-                        sys.stderr.write(f"{e}\n")
-                        sys.stderr.flush()
-                        exception = e
-                else:
-                    try:
-                        with tracer.start_as_current_span(
-                            DB_WRITE_EXECUTE, **write_span_kwargs
-                        ) as span:
-                            span.set_attribute(
-                                ISOLATED_CONNECTION,
-                                task.isolated_connection,
-                            )
-                            span.set_attribute(TRANSACTION, task.transaction)
-                            if task.transaction:
-                                with conn:
-                                    conn.execute("BEGIN IMMEDIATE")
-                                    result = task.fn(conn)
-                            else:
-                                result = task.fn(conn)
-                    except Exception as e:  # noqa: BLE001
-                        sys.stderr.write(f"{e}\n")
-                        sys.stderr.flush()
-                        exception = e
-                _deliver_write_result(task, result, exception)
-            finally:
-                if token is not None:
-                    otel_context_api.detach(token)
+                try:
+                    if task.transaction:
+                        with conn:
+                            conn.execute("BEGIN IMMEDIATE")
+                            result = task.fn(conn)
+                    else:
+                        result = task.fn(conn)
+                except Exception as e:  # noqa: BLE001
+                    sys.stderr.write(f"{e}\n")
+                    sys.stderr.flush()
+                    exception = e
+            _deliver_write_result(task, result, exception)
 
     async def execute_fn(self, fn):
         self._check_not_closed()
@@ -677,28 +534,7 @@ class Database:
 
         with self._pending_execute_futures_lock:
             self._check_not_closed()
-            # A fresh copy_context() is required per submit (not one shared
-            # copy reused across calls): concurrent execution of the same
-            # Context raises "RuntimeError: cannot enter context ...
-            # already entered". This propagates the caller's otel context
-            # (e.g. the enclosing db.query span) onto the worker thread.
-            #
-            # copy_context() is not selective: it also carries Datasette's own
-            # ContextVars - _skip_permission_checks and _permission_check_cache
-            # (datasette/permissions.py) and _in_datasette_client (app.py) -
-            # into worker threads, where they previously took their defaults.
-            # That is safe, for two reasons. Nothing reads them on a worker
-            # thread: the permission code that reads the first two is async and
-            # only ever runs on the event loop. And Context.run() restores the
-            # thread's previous context when the callable returns, so a value
-            # cannot outlive the submit that carried it and reach the next task
-            # on this shared pool - "skip permission checks" in particular can
-            # never bleed from one request into another's query. Where a value
-            # would be read - a plugin calling datasette.in_client() from inside
-            # an execute_fn callable - seeing the submitting request's value is
-            # the more accurate answer, not a leak.
-            ctx = contextvars.copy_context()
-            future = self.ds.executor.submit(ctx.run, in_thread)
+            future = self.ds.executor.submit(in_thread)
             self._pending_execute_futures.add(future)
         future.add_done_callback(self._remove_pending_execute_future)
         return await asyncio.wrap_future(future)
@@ -711,143 +547,48 @@ class Database:
         custom_time_limit=None,
         page_size=None,
         log_sql_errors=True,
-        table=None,
     ):
-        """Executes sql against db_name in a thread
-
-        `table`, if passed, is recorded as the `db.collection.name` span
-        attribute. It exists for callers that already know which table the
-        query targets - the table and row views - and is never derived from
-        `sql` itself: deriving it would be a parse, and on an instance where
-        anyone can create a table the resulting value set has no ceiling.
-        """
+        """Executes sql against db_name in a thread"""
         self._check_not_closed()
         page_size = page_size or self.ds.page_size
-        time_limit_ms = self.ds.sql_time_limit_ms
-        # A caller that hands in a budget shorter than the instance-wide
-        # sql_time_limit_ms is saying "this may not finish, and that is an
-        # answer I can use" - and every such caller in core does treat the
-        # timeout as normal: table_counts() stores None per table, facet
-        # suggestion moves on to the next column, autocomplete falls back to a
-        # prefix query. Those timeouts are therefore not span errors. Without
-        # this, the homepage alone emits one red span per table (it counts
-        # every table under a 10ms budget) on every single hit.
-        #
-        # A query that runs out the instance-wide limit is a different event -
-        # nobody asked for a short budget, so it stays an error.
-        timeout_expected = bool(custom_time_limit) and custom_time_limit < time_limit_ms
-        if timeout_expected:
-            time_limit_ms = custom_time_limit
 
         def sql_operation_in_thread(conn):
-            # This span is created inside the worker thread. Its parent is
-            # resolved from the ambient otel context, which was propagated
-            # onto this thread via copy_context() at the executor.submit()
-            # boundary in execute_fn() (or run_in_executor() for immutable
-            # databases) - so it parents correctly to the enclosing
-            # db.query span despite running on a different thread.
-            #
-            # Exception handling is explicit rather than left to the context
-            # manager's flags, which apply to every exception type alike. This
-            # span needs to tell two apart: an expected timeout is never an
-            # error, while a genuine SQL failure is one unless the caller
-            # passed log_sql_errors=False, meaning it was probing and treats
-            # failure as an expected answer. Without the latter, facet
-            # suggestion marks two spans per text column as failed on every
-            # table page; without the former, so does every homepage hit.
-            with tracer.start_as_current_span(
-                DB_QUERY_EXECUTE,
-                record_exception=False,
-                set_status_on_exception=False,
-            ) as execute_span:
+            time_limit_ms = self.ds.sql_time_limit_ms
+            if custom_time_limit and custom_time_limit < time_limit_ms:
+                time_limit_ms = custom_time_limit
+
+            with sqlite_timelimit(conn, time_limit_ms):
                 try:
-                    with sqlite_timelimit(conn, time_limit_ms):
-                        try:
-                            cursor = conn.cursor()
-                            cursor.execute(sql, params if params is not None else {})
-                            max_returned_rows = self.ds.max_returned_rows
-                            if max_returned_rows == page_size:
-                                max_returned_rows += 1
-                            if max_returned_rows and truncate:
-                                rows = cursor.fetchmany(max_returned_rows + 1)
-                                truncated = len(rows) > max_returned_rows
-                                rows = rows[:max_returned_rows]
-                            else:
-                                rows = cursor.fetchall()
-                                truncated = False
-                        except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:
-                            if e.args == ("interrupted",):
-                                raise QueryInterrupted(e, sql, params)
-                            if log_sql_errors:
-                                sys.stderr.write(
-                                    f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n"
-                                )
-                                sys.stderr.flush()
-                            raise
-                except QueryInterrupted as e:
-                    if not timeout_expected:
-                        execute_span.record_exception(e)
-                        execute_span.set_status(Status(StatusCode.ERROR, str(e)))
-                    raise
-                except Exception as e:
+                    cursor = conn.cursor()
+                    cursor.execute(sql, params if params is not None else {})
+                    max_returned_rows = self.ds.max_returned_rows
+                    if max_returned_rows == page_size:
+                        max_returned_rows += 1
+                    if max_returned_rows and truncate:
+                        rows = cursor.fetchmany(max_returned_rows + 1)
+                        truncated = len(rows) > max_returned_rows
+                        rows = rows[:max_returned_rows]
+                    else:
+                        rows = cursor.fetchall()
+                        truncated = False
+                except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:
+                    if e.args == ("interrupted",):
+                        raise QueryInterrupted(e, sql, params)
                     if log_sql_errors:
-                        execute_span.record_exception(e)
-                        execute_span.set_status(Status(StatusCode.ERROR, str(e)))
+                        sys.stderr.write(
+                            f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n"
+                        )
+                        sys.stderr.flush()
                     raise
 
-                if truncate:
-                    return Results(rows, truncated, cursor.description)
+            if truncate:
+                return Results(rows, truncated, cursor.description)
 
-                else:
-                    return Results(rows, False, cursor.description)
+            else:
+                return Results(rows, False, cursor.description)
 
-        # Exception handling is explicit rather than left to the context
-        # manager's defaults, so that callers passing log_sql_errors=False
-        # can be honoured - see the comment on the generic handler below.
-        with tracer.start_as_current_span(
-            DB_QUERY,
-            kind=DB_QUERY.kind,
-            record_exception=False,
-            set_status_on_exception=False,
-        ) as span:
-            span.set_attribute(DB_SYSTEM, "sqlite")
-            span.set_attribute(DB_NAMESPACE, self.name)
-            span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
-            span.set_attribute(TIME_LIMIT_MS, time_limit_ms)
-            operation_name = sql_operation_name(sql)
-            if operation_name:
-                span.set_attribute(DB_OPERATION_NAME, operation_name)
-            if table:
-                span.set_attribute(DB_COLLECTION_NAME, table)
-            if params:
-                span.set_attribute(PARAM_COUNT, len(params))
-            try:
-                results = await self.execute_fn(sql_operation_in_thread)
-            except QueryInterrupted as e:
-                # datasette.interrupted is set either way - it is the
-                # signal worth having. Only the ERROR status is
-                # conditional; see the timeout_expected comment above.
-                span.set_attribute(INTERRUPTED, True)
-                if not timeout_expected:
-                    span.set_status(Status(StatusCode.ERROR, str(e)))
-                    span.record_exception(e)
-                raise
-            except Exception as e:
-                # log_sql_errors=False means the caller is probing and
-                # treats failure as an expected answer, not an error.
-                # Facet suggestion is the big one: it runs json_type()
-                # against every column precisely to find out which ones
-                # raise, so a table with N text columns would otherwise
-                # mark N queries per page as failed - burying real errors
-                # and setting off any alerting based on span status.
-                if log_sql_errors:
-                    span.record_exception(e)
-                    span.set_status(Status(StatusCode.ERROR, str(e)))
-                else:
-                    span.set_attribute(SQL_ERROR_SUPPRESSED, True)
-                raise
-            span.set_attribute(TRUNCATED, results.truncated)
-            span.set_attribute(ROWS_RETURNED, len(results.rows))
+        with trace("sql", database=self.name, sql=sql.strip(), params=params):
+            results = await self.execute_fn(sql_operation_in_thread)
         return results
 
     @property
@@ -1040,6 +781,17 @@ class Database:
 
         return hidden_tables
 
+    async def derived_table_dependencies(self):
+        """Return implementation tables and the tables they derive from."""
+        schema_version = (await self.execute("PRAGMA schema_version")).first()[0]
+        if (
+            self._cached_derived_table_dependencies is None
+            or self._cached_derived_table_dependencies[0] != schema_version
+        ):
+            dependencies = await self.execute_fn(sqlite_derived_table_dependencies)
+            self._cached_derived_table_dependencies = (schema_version, dependencies)
+        return self._cached_derived_table_dependencies[1]
+
     async def view_names(self):
         results = await self.execute("select name from sqlite_master where type='view'")
         return [r[0] for r in results.rows]
@@ -1135,28 +887,16 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event):
 
 class WriteTask:
     __slots__ = (
-        "block",
-        "enqueued_at_ns",
         "fn",
         "isolated_connection",
         "loop",
-        "otel_context",
         "reply_future",
         "task_id",
         "transaction",
     )
 
     def __init__(
-        self,
-        fn,
-        task_id,
-        loop,
-        reply_future,
-        isolated_connection,
-        transaction,
-        otel_context,
-        enqueued_at_ns,
-        block,
+        self, fn, task_id, loop, reply_future, isolated_connection, transaction
     ):
         self.fn = fn
         self.task_id = task_id
@@ -1164,14 +904,6 @@ class WriteTask:
         self.reply_future = reply_future
         self.isolated_connection = isolated_connection
         self.transaction = transaction
-        self.otel_context = otel_context
-        self.enqueued_at_ns = enqueued_at_ns
-        # Whether the enqueueing caller awaits the reply future. Decides how
-        # `_execute_writes` relates this task's spans to `otel_context`:
-        # parent (block=True) or span-link target (block=False). See the
-        # comment at the WriteTask construction site in
-        # `_send_to_write_thread`.
-        self.block = block
 
 
 def _deliver_write_result(task, result, exception):

datasette/default_column_types.py

@@ -6,6 +6,17 @@ import markupsafe
 from datasette import hookimpl
 from datasette.column_types import ColumnType, SQLiteType
 
+_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE)
+
+
+def _normalize_http_url(value):
+    if not isinstance(value, str):
+        return None
+    normalized = value.strip()
+    if not _HTTP_URL_RE.fullmatch(normalized):
+        return None
+    return normalized
+
 
 class UrlColumnType(ColumnType):
     name = "url"
@@ -15,7 +26,10 @@ class UrlColumnType(ColumnType):
     async def render_cell(self, value, column, table, database, datasette, request):
         if not value or not isinstance(value, str):
             return None
-        escaped = markupsafe.escape(value.strip())
+        normalized = _normalize_http_url(value)
+        if normalized is None:
+            return markupsafe.escape(value.strip())
+        escaped = markupsafe.escape(normalized)
         return markupsafe.Markup(f'<a href="{escaped}">{escaped}</a>')
 
     async def validate(self, value, datasette):
@@ -23,7 +37,7 @@ class UrlColumnType(ColumnType):
             return None
         if not isinstance(value, str):
             return "URL must be a string"
-        if not re.match(r"^https?://\S+$", value.strip()):
+        if _normalize_http_url(value) is None:
             return "Invalid URL"
         return None
 

datasette/default_permissions/config.py

@@ -92,6 +92,13 @@ class ConfigPermissionProcessor:
             # Tables implicitly reference their parent databases
             self.restricted_databases.update(db for db, _ in self.restricted_tables)
 
+        # Resolve identity keys once per action, rather than scanning the
+        # restriction allowlist for every configured table's allow block.
+        self.restricted_table_keys = {
+            (db, self.action_obj.normalize_child(table) if self.action_obj else table)
+            for db, table in self.restricted_tables
+        }
+
     def evaluate_allow_block(self, allow_block: Any) -> bool | None:
         """Evaluate an allow block against the current actor."""
         if allow_block is None:
@@ -125,8 +132,10 @@ class ConfigPermissionProcessor:
         if parent:
             table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {})
             if child:
-                table_actions = table_restrictions.get(child, [])
-                if self.action_checks.intersection(table_actions):
+                child_key = (
+                    self.action_obj.normalize_child(child) if self.action_obj else child
+                )
+                if (parent, child_key) in self.restricted_table_keys:
                     return True
             else:
                 # Parent query should proceed if any child in this database is allowlisted

datasette/default_permissions/restrictions.py

@@ -185,11 +185,15 @@ def restrictions_allow_action(
     # Check table/resource level
     if resource is not None and not isinstance(resource, str) and len(resource) == 2:
         database, table = resource
-        table_allowed = restrictions.get("r", {}).get(database, {}).get(table)
-        if table_allowed is not None:
-            assert isinstance(table_allowed, list)
-            if to_check.intersection(table_allowed):
-                return True
+        action_obj = datasette.actions.get(action)
+        normalize = action_obj.normalize_child if action_obj else lambda name: name
+        for table_name, table_allowed in (
+            restrictions.get("r", {}).get(database, {}).items()
+        ):
+            if normalize(table_name) == normalize(table):
+                assert isinstance(table_allowed, list)
+                if to_check.intersection(table_allowed):
+                    return True
 
     # This action is not explicitly allowed, so reject it
     return False

datasette/default_permissions/sqlite_statistics.py

@@ -0,0 +1,25 @@
+"""Default table-access policy for SQLite optimizer statistics."""
+
+import json
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+
+@hookimpl
+def permission_resources_sql(action):
+    if action != "view-table":
+        return None
+    return PermissionSQL(
+        sql="""
+            SELECT database_name AS parent, value AS child, 0 AS allow,
+                'SQLite statistics tables are denied by default' AS reason
+            FROM catalog_databases
+            CROSS JOIN json_each(:sqlite_statistics_names)
+        """,
+        params={
+            "sqlite_statistics_names": json.dumps(
+                ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"]
+            )
+        },
+    )

datasette/filters.py

@@ -2,7 +2,7 @@ import json
 from typing import ClassVar
 
 from datasette import hookimpl
-from datasette.resources import DatabaseResource
+from datasette.resources import DatabaseResource, TableResource
 from datasette.utils.asgi import BadRequest
 from datasette.views.base import DatasetteError
 
@@ -51,13 +51,20 @@ def search_filters(request, database, table, datasette):
         human_descriptions = []
         extra_context = {}
 
-        # Figure out which fts_table to use
+        # Figure out which trusted fts_table to use. Query string parameters can
+        # repeat this mapping (for backwards compatibility), but must not select
+        # a different table or primary key.
         table_metadata = await datasette.table_config(database, table)
         db = datasette.get_database(database)
-        fts_table = request.args.get("_fts_table")
-        fts_table = fts_table or table_metadata.get("fts_table")
+        fts_table = table_metadata.get("fts_table")
         fts_table = fts_table or await db.fts_table(table)
-        fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid"))
+        fts_pk = table_metadata.get("fts_pk", "rowid")
+        requested_fts_table = request.args.get("_fts_table")
+        requested_fts_pk = request.args.get("_fts_pk")
+        if (requested_fts_table and requested_fts_table != fts_table) or (
+            requested_fts_pk and requested_fts_pk != fts_pk
+        ):
+            raise BadRequest("Invalid _fts_table or _fts_pk")
         search_args = {
             key: request.args[key]
             for key in request.args
@@ -75,6 +82,11 @@ def search_filters(request, database, table, datasette):
         extra_context["supports_search"] = bool(fts_table)
 
         if fts_table and search_args:
+            await datasette.ensure_permission(
+                action="view-table",
+                resource=TableResource(database=database, table=fts_table),
+                actor=request.actor,
+            )
             if "_search" in search_args:
                 # Simple ?_search=xxx
                 search = search_args["_search"]
@@ -135,6 +147,11 @@ def through_filters(request, database, table, datasette):
                 through_table = through_data["table"]
                 other_column = through_data["column"]
                 value = through_data["value"]
+                await datasette.ensure_permission(
+                    action="view-table",
+                    resource=TableResource(database=database, table=through_table),
+                    actor=request.actor,
+                )
                 db = datasette.get_database(database)
                 outgoing_foreign_keys = await db.foreign_keys_for_table(through_table)
                 fk_to_us = next(

datasette/permissions.py

@@ -3,6 +3,10 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Any, NamedTuple
 
+_SQLITE_IDENTIFIER_CASE = str.maketrans(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
+)
+
 # Context variable to track when permission checks should be skipped
 _skip_permission_checks = contextvars.ContextVar(
     "skip_permission_checks", default=False
@@ -49,6 +53,15 @@ class Resource(ABC):
     # Class-level metadata (subclasses must define these)
     name: str = None  # e.g., "table", "database", "model"
     parent_class: type["Resource"] | None = None  # e.g., DatabaseResource for tables
+    case_insensitive_child: bool = False
+
+    @classmethod
+    def normalize_child(cls, child: str | None) -> str | None:
+        """Return a comparison key without changing the resource's display name."""
+        if cls.case_insensitive_child and child is not None:
+            # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold.
+            return child.translate(_SQLITE_IDENTIFIER_CASE)
+        return child
 
     # Instance-level optional extra attributes
     reasons: list[str] | None = None
@@ -146,6 +159,11 @@ class Action:
     resource_class: type[Resource] | None = None
     also_requires: str | None = None  # Optional action name that must also be allowed
 
+    def normalize_child(self, child: str | None) -> str | None:
+        if self.resource_class is None:
+            return child
+        return self.resource_class.normalize_child(child)
+
     @property
     def takes_parent(self) -> bool:
         """

datasette/plugins.py

@@ -18,6 +18,7 @@ DEFAULT_PLUGINS = (
     "datasette.actor_auth_cookie",
     "datasette.default_permissions",
     "datasette.default_permissions.tokens",
+    "datasette.default_permissions.sqlite_statistics",
     "datasette.default_actions",
     "datasette.default_column_types",
     "datasette.default_magic_parameters",

datasette/resources.py

@@ -25,6 +25,7 @@ class TableResource(Resource):
 
     name = "table"
     parent_class = DatabaseResource
+    case_insensitive_child = True
 
     def __init__(self, database: str, table: str):
         super().__init__(parent=database, child=table)

datasette/static/column-chooser.js

@@ -472,11 +472,13 @@ class ColumnChooser extends HTMLElement {
           <span class="drag-item-check">
             <input type="checkbox" ${this._checked.has(col) ? "checked" : ""}>
           </span>
-          <span class="drag-item-label">${col}</span>
+          <span class="drag-item-label"></span>
         </label>
         <div class="drop-indicator"></div>
       `;
 
+      li.querySelector(".drag-item-label").textContent = col;
+
       li.querySelector("input").addEventListener("change", (e) => {
         e.target.checked ? this._checked.add(col) : this._checked.delete(col);
         this._updateCounts();

datasette/static/json-format-highlight-1.0.1.js

@@ -1,56 +0,0 @@
-/*
-https://github.com/luyilin/json-format-highlight
-From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js
-MIT Licensed
-*/
-(function (global, factory) {
-  typeof exports === "object" && typeof module !== "undefined"
-    ? (module.exports = factory())
-    : typeof define === "function" && define.amd
-      ? define(factory)
-      : (global.jsonFormatHighlight = factory());
-})(this, function () {
-  "use strict";
-
-  var defaultColors = {
-    keyColor: "dimgray",
-    numberColor: "lightskyblue",
-    stringColor: "lightcoral",
-    trueColor: "lightseagreen",
-    falseColor: "#f66578",
-    nullColor: "cornflowerblue",
-  };
-
-  function index(json, colorOptions) {
-    if (colorOptions === void 0) colorOptions = {};
-
-    if (!json) {
-      return;
-    }
-    if (typeof json !== "string") {
-      json = JSON.stringify(json, null, 2);
-    }
-    var colors = Object.assign({}, defaultColors, colorOptions);
-    json = json.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
-    return json.replace(
-      /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g,
-      function (match) {
-        var color = colors.numberColor;
-        if (/^"/.test(match)) {
-          color = /:$/.test(match) ? colors.keyColor : colors.stringColor;
-        } else {
-          color = /true/.test(match)
-            ? colors.trueColor
-            : /false/.test(match)
-              ? colors.falseColor
-              : /null/.test(match)
-                ? colors.nullColor
-                : color;
-        }
-        return '<span style="color: ' + color + '">' + match + "</span>";
-      },
-    );
-  }
-
-  return index;
-});

datasette/telemetry.py

@@ -1,347 +0,0 @@
-"""
-OpenTelemetry integration for Datasette core.
-
-Core depends on `opentelemetry-api` only. It never creates a
-`TracerProvider`, never configures an exporter, and never touches
-sampling - that is the responsibility of whoever is running Datasette
-(an `opentelemetry-instrument` agent, a future plugin, or a test
-harness).
-
-With no provider installed every span produced here is a
-`NonRecordingSpan`. That is not free - a table page emits ~58 spans -
-but it is below what an end-to-end page benchmark can resolve: measured
-across 15 runs of a 5,000-row table page, the median moved 9.80ms to
-9.98ms while run-to-run spread was 1.4ms. Installing an SDK provider is
-what costs something measurable.
-"""
-
-import re
-
-from opentelemetry import trace as otel_trace
-from opentelemetry.propagate import extract
-from opentelemetry.propagators.textmap import Getter
-from opentelemetry.trace import SpanKind, Status, StatusCode
-
-from .telemetry_registry import (
-    ERROR_TYPE,
-    HTTP_REQUEST_METHOD,
-    HTTP_RESPONSE_STATUS_CODE,
-    SERVER_ADDRESS,
-    URL_PATH,
-    URL_SCHEME,
-    USER_AGENT_ORIGINAL,
-)
-from .version import __version__
-
-# The semantic-convention version whose spellings this instrumentation
-# actually emits. Deliberately NOT the latest release.
-#
-# A schema URL is a machine-readable claim: a consumer doing schema
-# translation replays the renames between the declared version and the one
-# it wants, so the claim has to name the version whose spellings are on the
-# wire. A wrong one makes translation wrong rather than merely uninformative.
-#
-# Datasette emits `db.system`, which was renamed to `db.system.name` in
-# semconv 1.30.0. Everything else it emits (`db.namespace`, `db.query.text`,
-# `db.operation.name`, `db.collection.name`) has been current since 1.26.0.
-# So 1.29.0 is the highest version at which every name emitted here is the
-# current spelling. Everything under `datasette.*` is Datasette's own and
-# outside semconv, so it is unaffected either way.
-#
-# Declaring 1.43.0 would be false about `db.system`, and would actively STOP
-# a consumer translating it forward, because it asserts the rename already
-# happened. Bump this deliberately, in the same commit as the attribute
-# renames it implies - it is a claim about the names, not decoration.
-SCHEMA_URL = "https://opentelemetry.io/schemas/1.29.0"
-
-tracer = otel_trace.get_tracer("datasette", __version__, schema_url=SCHEMA_URL)
-
-MAX_SQL_LENGTH = 2048
-
-
-def sql_attribute(sql: str) -> str:
-    "Truncate SQL text so it is safe to attach to a span as an attribute."
-    sql = sql.strip()
-    if len(sql) <= MAX_SQL_LENGTH:
-        return sql
-    return sql[:MAX_SQL_LENGTH] + "…[truncated]"
-
-
-# db.operation.name is the leading keyword of a statement matched against a
-# fixed allowlist - deliberately not a parse.
-#
-# This runs against arbitrary user-supplied SQL (the `?sql=` query string,
-# canned queries, anything typed into the query editor), and the attribute is
-# a candidate dimension on a query-duration metric in a later phase. A metric
-# series is keyed by its attribute values, so echoing back an arbitrary first
-# token would let one visitor's typo mint a new, permanent series. The
-# allowlist bounds that at a fixed, small set regardless of what anyone sends.
-DB_OPERATION_ALLOWLIST = frozenset(
-    {
-        "SELECT",
-        "INSERT",
-        "UPDATE",
-        "DELETE",
-        "CREATE",
-        "DROP",
-        "ALTER",
-        "PRAGMA",
-        "EXPLAIN",
-        "REPLACE",
-        "VACUUM",
-        "ANALYZE",
-        "WITH",
-    }
-)
-
-_LEADING_KEYWORD = re.compile(r"^\s*([A-Za-z]+)")
-
-
-def sql_operation_name(sql: str) -> str | None:
-    """
-    The statement's leading keyword, if it is one we recognise.
-
-    Returns None - never a guess - for anything not on the allowlist,
-    including a statement that opens with a comment or with punctuation such
-    as the "(" of a parenthesised SELECT.
-
-    Known limitation: a statement beginning with a CTE reports `WITH` rather
-    than the operation inside it, and a substantial share of Datasette's own
-    reads take that form. Extracting more than the leading keyword means
-    handling comment stripping, parenthesised `(SELECT ...) UNION` and
-    compound names like `CREATE TABLE` - each a special case a hand-rolled
-    matcher would accrete and eventually get wrong. Omitting a name beats
-    guessing at one.
-
-    Only safe to call with a single statement: `execute_write_script()` runs
-    several separated by semicolons, and semantic conventions say
-    `db.operation.name` "SHOULD NOT be extracted from db.query.text, when the
-    database system supports query text with multiple operations in non-batch
-    operations" - so that call site does not use this at all rather than
-    reporting only the first statement's operation.
-    """
-    match = _LEADING_KEYWORD.match(sql)
-    if not match:
-        return None
-    keyword = match.group(1).upper()
-    if keyword in DB_OPERATION_ALLOWLIST:
-        return keyword
-    return None
-
-
-# --- The HTTP request span ------------------------------------------------
-
-
-class _ScopeHeadersGetter(Getter):
-    """
-    Read W3C trace context out of an ASGI scope's headers.
-
-    `scope["headers"]` is a list of `(bytes, bytes)` pairs, lowercased by the
-    server per the ASGI spec - but `.lower()` is applied again here because
-    that is a spec promise about servers, not something this process
-    controls. Header bytes are latin-1 by RFC 9110.
-    """
-
-    def get(self, carrier, key):
-        wanted = key.lower().encode("latin-1")
-        values = [v.decode("latin-1") for k, v in carrier if k.lower() == wanted]
-        return values or None
-
-    def keys(self, carrier):
-        return [k.decode("latin-1") for k, _ in carrier]
-
-
-_HEADERS_GETTER = _ScopeHeadersGetter()
-
-
-# An unclamped method is an unbounded dimension a client controls: anyone can
-# send `FOO / HTTP/1.1`. Semantic conventions say map anything unrecognised to
-# `_OTHER`. These nine are the methods of RFC 9110 plus PATCH (RFC 5789).
-_KNOWN_METHODS = frozenset(
-    {"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"}
-)
-
-
-def clamp_http_method(method):
-    "The request method if it is one we recognise, else ``_OTHER``."
-    method = (method or "").upper()
-    return method if method in _KNOWN_METHODS else "_OTHER"
-
-
-def _first_header(headers, name):
-    "The first value of a header, decoded, or None."
-    for key, value in headers:
-        if key.lower() == name:
-            return value.decode("latin-1")
-    return None
-
-
-def _url_path(scope):
-    """
-    The request path, with any query string removed.
-
-    `raw_path` is preferred because it is the bytes the client sent, before
-    percent-decoding - Datasette routes on database and table names that can
-    contain encoded slashes, which `scope["path"]` has already collapsed.
-
-    The split on "?" is not decoration. The ASGI spec's `raw_path` excludes
-    the query string, and uvicorn honours that, but the name is used the
-    other way round elsewhere in this same dependency tree: httpx's
-    `URL.raw_path` is documented as "raw bytes of both the path and query".
-    A server that followed that reading would hand us `?sql=...` here, and
-    Datasette's query strings carry user-supplied SQL, which core never
-    records. A literal "?" cannot appear unencoded in a path, so the split
-    costs nothing when the server is well behaved.
-    """
-    raw_path = scope.get("raw_path")
-    if raw_path:
-        if isinstance(raw_path, bytes):
-            raw_path = raw_path.decode("latin-1")
-        return raw_path.split("?", 1)[0]
-    return scope.get("path", "")
-
-
-# The request span is handed to `DatasetteRouter.route_path` through the ASGI
-# scope rather than through `get_current_span()`, because by the time routing
-# happens the current span may well be something else: a plugin
-# `asgi_wrapper()` runs *inside* this middleware, and an instrumented one makes
-# its own span current for the whole request. Reading the current span there
-# would set `http.route` on that plugin's span - and rename it - while leaving
-# the actual request span without the one attribute a trace UI groups by. Not
-# hypothetical: an ordinary tracing plugin triggers it.
-#
-# Namespaced per the ASGI spec's rules for extension keys. Absent when the span
-# is not recording, which is exactly when the router should skip the work too.
-REQUEST_SPAN_SCOPE_KEY = "datasette.telemetry.request_span"
-
-
-def request_span(scope):
-    """
-    The recording request span for an ASGI scope, or None.
-
-    Falls back to the current span so that a `DatasetteRouter` running under
-    some other instrumentation - one that started a SERVER span but of course
-    knows nothing about this scope key - still gets enriched.
-    """
-    span = scope.get(REQUEST_SPAN_SCOPE_KEY)
-    if span is None:
-        span = otel_trace.get_current_span()
-    # is_recording(), not `get_span_context().is_valid`: with no provider but
-    # an inbound `traceparent`, the API's NoOpTracer hands back a
-    # NonRecordingSpan carrying the *remote* context, which is perfectly valid
-    # and still records nothing.
-    return span if span.is_recording() else None
-
-
-class TelemetryMiddleware:
-    """
-    One `SpanKind.SERVER` span per HTTP request.
-
-    Mounted outermost in `Datasette.app()`, so every other span raised while
-    serving a request - database queries, plugin middleware, startup work on
-    a cold ASGI-hosted deployment - has somewhere to belong instead of
-    becoming its own root trace.
-
-    Deliberately much smaller than `opentelemetry-instrumentation-asgi`,
-    which needs several hundred lines of deferred-end machinery for
-    applications that return before their body is sent. Datasette does not:
-    `DatasetteRouter.route_path` awaits `response.asgi_send(send)`, and for a
-    streaming CSV export `AsgiStream.asgi_send` runs the generator inline.
-    All of it happens inside the single `await self.app(...)` below, so
-    ending the span in a `finally` covers the response body too.
-    """
-
-    def __init__(self, app):
-        self.app = app
-
-    async def __call__(self, scope, receive, send):
-        # First, before anything else: `AsgiLifespan` is *inside* this
-        # middleware, so lifespan startup and shutdown have to pass through
-        # untouched or the server never starts. Same for websockets.
-        if scope["type"] != "http":
-            await self.app(scope, receive, send)
-            return
-        headers = scope.get("headers") or []
-        # The *global* propagator, deliberately: it leaves the operator in
-        # control with no Datasette-specific setting - OTEL_PROPAGATORS=none
-        # disables extraction entirely, OTEL_PROPAGATORS=tracecontext drops
-        # baggage - and core configuring propagation itself would be the same
-        # mistake as core configuring sampling.
-        context = extract(headers, getter=_HEADERS_GETTER)
-        method = clamp_http_method(scope.get("method", ""))
-        # The method, not the URL: a span name has to be low cardinality, and
-        # the method is what is known out here at the edge, before any routing
-        # has happened.
-        with tracer.start_as_current_span(
-            method, context=context, kind=SpanKind.SERVER
-        ) as span:
-            if not span.is_recording():
-                # No provider installed, or a sampler dropped this trace.
-                # Everything below would be discarded, so skip building the
-                # `send` wrapper and let a default install pay almost
-                # nothing. Note this cannot be `get_span_context().is_valid`:
-                # with no provider but an inbound `traceparent`, the API's
-                # NoOpTracer returns a NonRecordingSpan carrying the *remote*
-                # context, which is perfectly valid and still records nothing.
-                await self.app(scope, receive, send)
-                return
-            span.set_attribute(HTTP_REQUEST_METHOD, method)
-            span.set_attribute(URL_PATH, _url_path(scope))
-            scheme = scope.get("scheme")
-            if scheme:
-                span.set_attribute(URL_SCHEME, scheme)
-            host = _first_header(headers, b"host")
-            if host:
-                span.set_attribute(SERVER_ADDRESS, host)
-            user_agent = _first_header(headers, b"user-agent")
-            if user_agent:
-                span.set_attribute(USER_AGENT_ORIGINAL, user_agent)
-
-            # A copy, not a mutation: the scope belongs to the server, and
-            # every other layer in Datasette extends it the same way.
-            scope = dict(scope, **{REQUEST_SPAN_SCOPE_KEY: span})
-
-            # The status cannot be read off a Response object: `asgi_static`,
-            # the favicon route, `AsgiStream` and `AsgiFileDownload` all call
-            # `send` directly and never build one. Wrapping `send` is the only
-            # thing that sees every response, including the 404 and 500
-            # handlers.
-            status_holder = {}
-
-            async def wrapped_send(message):
-                if (
-                    message["type"] == "http.response.start"
-                    and "status" not in status_holder
-                ):
-                    status_holder["status"] = message["status"]
-                await send(message)
-
-            escaped = False
-            try:
-                # Positional (scope, receive, send) throughout this codebase -
-                # `wrapped_send` is the third argument. `receive` is passed
-                # through unwrapped.
-                await self.app(scope, receive, wrapped_send)
-            except BaseException as exception:
-                # BaseException, not Exception: `route_path` turns almost
-                # everything into a 500 itself, but `asyncio.CancelledError`
-                # on client disconnect is a BaseException its `except
-                # Exception` deliberately does not catch.
-                escaped = True
-                span.set_attribute(ERROR_TYPE, type(exception).__name__)
-                span.set_status(Status(StatusCode.ERROR, str(exception)))
-                raise
-            finally:
-                status = status_holder.get("status")
-                if status is not None:
-                    span.set_attribute(HTTP_RESPONSE_STATUS_CODE, status)
-                    # 4xx is NOT an error for a SERVER span per semantic
-                    # conventions - the client made the mistake, not us.
-                    #
-                    # `not escaped` because this block still runs when an
-                    # exception is on its way out, and a response can have
-                    # started before it: the exception's class name is more
-                    # use than the string "500", so it wins.
-                    if status >= 500 and not escaped:
-                        span.set_status(Status(StatusCode.ERROR))
-                        span.set_attribute(ERROR_TYPE, str(status))

datasette/telemetry_registry.py

@@ -1,388 +0,0 @@
-"""
-The single source of truth for every span and span attribute that Datasette
-core emits.
-
-Three things read this module, which is the point of it existing:
-
-1. **The instrumentation itself.** `Attribute` and `SpanName` subclass `str`,
-   so a registry entry *is* the string OpenTelemetry wants. Call sites pass
-   `DB_NAMESPACE` where they used to pass `"db.namespace"` - no wrapper API
-   over the OTel calls, no parallel structure to keep in step, and a typo is
-   now an `ImportError` instead of a silently misnamed attribute.
-
-2. **The documentation.** `docs/telemetry_doc.py` renders the span reference
-   in `docs/internals.rst` from these definitions using cog, and
-   `cog --check` runs in CI - so the docs cannot drift from the code.
-
-3. **A conformance test.** `tests/test_telemetry_registry.py` makes real
-   requests, collects every span and attribute actually emitted, and compares
-   both directions: emitted-but-unregistered catches instrumentation added
-   without documentation, registered-but-never-emitted catches documentation
-   describing something that no longer exists. Neither the type system nor
-   the generated docs can catch that second case.
-"""
-
-from opentelemetry.trace import SpanKind
-
-
-class Attribute(str):
-    """
-    A span attribute key, carrying its own documentation.
-
-    Subclasses `str` so it can be handed straight to `set_attribute()`.
-    """
-
-    __slots__ = ("description", "optional")
-
-    def __new__(cls, name, description, optional=False):
-        self = super().__new__(cls, name)
-        self.description = description
-        self.optional = optional
-        return self
-
-    def __repr__(self):
-        return f"Attribute({str(self)!r})"
-
-
-class SpanName(str):
-    "A span name, carrying its documentation and the attributes it may set."
-
-    __slots__ = ("attributes", "description", "dynamic", "kind", "prefix")
-
-    def __new__(
-        cls,
-        name,
-        description,
-        attributes=(),
-        prefix=False,
-        dynamic=False,
-        kind=SpanKind.INTERNAL,
-    ):
-        self = super().__new__(cls, name)
-        self.description = description
-        self.attributes = tuple(attributes)
-        # True for a span family whose emitted names carry a variable suffix,
-        # so the conformance test matches by prefix rather than equality.
-        # Nothing sets it yet.
-        self.prefix = prefix
-        # True when the emitted name is composed at runtime and shares no
-        # fixed prefix with the registry entry - the HTTP request span, whose
-        # name is the request method followed by the matched route. There is
-        # no substring of the entry that could be matched against the wire, so
-        # `span_for()` resolves these by span kind instead, and the entry's own
-        # string is a template written for a human reading the generated
-        # reference.
-        self.dynamic = dynamic
-        # SpanKind.INTERNAL by default - every span Datasette emits describes
-        # its own internal work. db.query is the one exception: it is a real
-        # database call, so semantic conventions (and trace UIs, which key
-        # their database styling off this) expect SpanKind.CLIENT.
-        self.kind = kind
-        return self
-
-    def __repr__(self):
-        return f"SpanName({str(self)!r})"
-
-
-# --- Attributes -----------------------------------------------------------
-#
-# Shared attributes are defined once and referenced by every span that sets
-# them, so "which spans carry db.namespace?" is answerable by grep.
-
-HTTP_REQUEST_METHOD = Attribute(
-    "http.request.method",
-    "The HTTP method, clamped to the nine methods RFC 9110 and RFC 5789 "
-    "define. Anything else is reported as ``_OTHER``: the method is a "
-    "client-controlled string, so echoing it back unbounded would be a "
-    "cardinality hazard.",
-)
-HTTP_RESPONSE_STATUS_CODE = Attribute(
-    "http.response.status_code",
-    "The status of the response, read from the ASGI ``http.response.start`` "
-    "message rather than from a :ref:`internals_response` object - several "
-    "views, including static files, file downloads and streaming CSV, send "
-    "that message themselves and never build one. Omitted if the connection "
-    "closed before anything was sent.",
-    optional=True,
-)
-HTTP_ROUTE = Attribute(
-    "http.route",
-    "The route the request matched, as the compiled regular expression "
-    "pattern Datasette routes with - for example "
-    "``/(?P<database>[^\\/\\.]+)/(?P<table>[^\\/\\.]+)(\\.(?P<format>\\w+))?$`` "
-    "for a table page. It is deliberately the pattern rather than a prettified "
-    "``/{database}/{table}`` template: the route table is fixed when the app "
-    "is built, so the pattern is exact, bounded and needs no parsing, whereas "
-    "the transform into something prettier accretes edge cases. Unlike "
-    "``url.path`` this is low cardinality, so it is the attribute to group by. "
-    "Omitted when no route matched - a 404 - which is also when the span name "
-    "falls back to the bare method.",
-    optional=True,
-)
-URL_PATH = Attribute(
-    "url.path",
-    "The path portion of the URL. The query string is deliberately **not** "
-    "recorded, on this or any other span: Datasette puts user-supplied SQL in "
-    "``?sql=`` and canned query parameters in the query string, so exporting "
-    "it by default would export exactly the data the rest of this "
-    "instrumentation is careful with.",
-)
-URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.")
-SERVER_ADDRESS = Attribute(
-    "server.address",
-    "The ``Host`` header. Client-controlled, so treat it as untrusted input "
-    "rather than as the identity of the server.",
-    optional=True,
-)
-USER_AGENT_ORIGINAL = Attribute(
-    "user_agent.original",
-    "The ``User-Agent`` header, verbatim. Omitted if the client sent none. "
-    "The client's IP address is deliberately not recorded: core records no "
-    "identifier that would tie a span to a person.",
-    optional=True,
-)
-ERROR_TYPE = Attribute(
-    "error.type",
-    "Set when the request failed: the exception class name if one escaped the "
-    "application, otherwise the status code as a string for a 5xx response. "
-    "A 4xx does **not** set this and does not set an error status - per "
-    "semantic conventions a client error is not a server span's failure.",
-    optional=True,
-)
-
-DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.")
-DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.")
-DB_QUERY_TEXT = Attribute(
-    "db.query.text",
-    "The SQL, truncated to 2048 characters. Never the parameter values.",
-)
-DB_OPERATION_NAME = Attribute(
-    "db.operation.name",
-    "The statement's leading keyword - ``SELECT``, ``INSERT``, ``CREATE``, and "
-    "so on - matched against a small fixed allowlist. Omitted rather than set "
-    "to an arbitrary value: the allowlist exists because this attribute is a "
-    "candidate dimension for a query-duration metric in a later phase, and "
-    "echoing an unrecognised first token from user-supplied SQL would be an "
-    "unbounded-cardinality hazard. Also omitted for "
-    "``execute_write_script()``, which runs multiple statements - per "
-    "semantic conventions, the operation name should not be extracted from "
-    "query text that can contain more than one operation. Note that a "
-    "statement beginning with a CTE reports ``WITH``, not the operation "
-    "inside it - a substantial share of Datasette's own reads take that "
-    "form. Resolving it further would mean parsing.",
-    optional=True,
-)
-DB_COLLECTION_NAME = Attribute(
-    "db.collection.name",
-    "The primary table, set only where the view already knows it - the table "
-    "and row pages. Omitted for arbitrary ``?sql=`` queries, where determining "
-    "the table would mean parsing the query.",
-    optional=True,
-)
-
-PARAM_COUNT = Attribute(
-    "datasette.param_count",
-    "Number of bound parameters. Recorded instead of the values themselves.",
-    optional=True,
-)
-PARAM_SETS = Attribute(
-    "datasette.param_sets",
-    "Number of parameter sets consumed by ``execute_write_many()``. Not a row "
-    "count - ``executemany()`` returns no rows. The parameter values "
-    "themselves are never recorded: that sequence can hold thousands of rows.",
-    optional=True,
-)
-TIME_LIMIT_MS = Attribute(
-    "datasette.time_limit_ms",
-    "The :ref:`setting_sql_time_limit_ms` value this query ran under. Set on "
-    "reads, which are the queries that time limit applies to.",
-    optional=True,
-)
-ROWS_RETURNED = Attribute(
-    "datasette.rows_returned",
-    "Number of rows a read returned. Set on the read path only, and only when "
-    "the read succeeded.",
-    optional=True,
-)
-TRUNCATED = Attribute(
-    "datasette.truncated",
-    "True if the result was cut short by :ref:`setting_max_returned_rows`.",
-    optional=True,
-)
-INTERRUPTED = Attribute(
-    "datasette.interrupted",
-    "True if the query was cancelled for exceeding the time limit. The span "
-    "status is also set to ``ERROR``, unless the caller asked for a budget "
-    "shorter than :ref:`setting_sql_time_limit_ms` - as table counts, facet "
-    "suggestion and autocomplete all do - in which case running out of time "
-    "is an expected answer rather than a failure and the status is left "
-    "unset.",
-    optional=True,
-)
-SQL_ERROR_SUPPRESSED = Attribute(
-    "datasette.sql_error_suppressed",
-    "True when the query failed but the caller passed ``log_sql_errors=False``, "
-    "meaning it was probing and treats failure as an expected answer. Facet "
-    "suggestion does this against every column.",
-    optional=True,
-)
-EXECUTESCRIPT = Attribute(
-    "datasette.executescript",
-    "True for ``execute_write_script()``, which runs multiple statements.",
-    optional=True,
-)
-EXECUTEMANY = Attribute(
-    "datasette.executemany",
-    "True for ``execute_write_many()``, which runs one statement against many "
-    "parameter sets.",
-    optional=True,
-)
-ISOLATED_CONNECTION = Attribute(
-    "datasette.isolated_connection",
-    "True if the write ran on its own connection rather than the shared write "
-    "connection.",
-)
-TRANSACTION = Attribute(
-    "datasette.transaction",
-    "False for statements such as ``VACUUM`` that cannot run inside a transaction.",
-)
-
-
-# --- Spans ----------------------------------------------------------------
-
-HTTP_REQUEST = SpanName(
-    "{http.request.method} {http.route}",
-    "One span per HTTP request, created by the outermost layer of the ASGI "
-    "stack - so plugin ``asgi_wrapper()`` middleware, CSRF protection and "
-    "every database span raised while serving the request all nest inside "
-    "it. Without it each of those would be its own root trace. The span name "
-    "is not a fixed string: it is the method followed by the matched route, "
-    "and just the method for a request that matched no route. The span starts "
-    "at the ASGI edge, before routing has happened, so it is named for the "
-    "method there and renamed once the route is known. "
-    "W3C ``traceparent`` and ``baggage`` headers are extracted using the "
-    "global propagator, so a request arriving from an already-traced caller "
-    "continues that trace; set ``OTEL_PROPAGATORS=none`` to turn that off, "
-    "and strip those headers at your proxy if your instance is public.",
-    (
-        HTTP_REQUEST_METHOD,
-        HTTP_ROUTE,
-        URL_PATH,
-        URL_SCHEME,
-        SERVER_ADDRESS,
-        USER_AGENT_ORIGINAL,
-        HTTP_RESPONSE_STATUS_CODE,
-        ERROR_TYPE,
-    ),
-    dynamic=True,
-    kind=SpanKind.SERVER,
-)
-
-DB_QUERY = SpanName(
-    "db.query",
-    "A SQL operation issued by Datasette, covering the full round trip "
-    "including any time spent queued for a thread.",
-    (
-        DB_SYSTEM,
-        DB_NAMESPACE,
-        DB_QUERY_TEXT,
-        DB_OPERATION_NAME,
-        DB_COLLECTION_NAME,
-        PARAM_COUNT,
-        PARAM_SETS,
-        TIME_LIMIT_MS,
-        ROWS_RETURNED,
-        TRUNCATED,
-        INTERRUPTED,
-        SQL_ERROR_SUPPRESSED,
-        EXECUTESCRIPT,
-        EXECUTEMANY,
-    ),
-    kind=SpanKind.CLIENT,
-)
-
-DB_QUERY_EXECUTE = SpanName(
-    "db.query.execute",
-    "The read executing inside a SQL worker thread. Child of ``db.query``; the "
-    "gap between the two is time spent waiting for a thread.",
-)
-
-DB_WRITE_QUEUE_WAIT = SpanName(
-    "db.write.queue_wait",
-    "Time a write spent waiting in its database's write queue before the write "
-    "thread picked it up. Child of ``db.query`` for a ``block=True`` write, "
-    "where the caller awaits the write and containment is accurate. For a "
-    "``block=False`` write the caller does not await it - the enqueueing "
-    "request *caused* the write without *containing* it, and the write's "
-    "spans can outlive the request's own - so this is a root span instead, "
-    "carrying an OpenTelemetry link back to the enqueueing span rather than "
-    "a parent. A link records causation without asserting containment, which "
-    "is exactly the distinction here.",
-)
-
-DB_WRITE_EXECUTE = SpanName(
-    "db.write.execute",
-    "The write executing on the write thread. Child of ``db.query`` for a "
-    "``block=True`` write; for ``block=False`` a root span with a link back "
-    "to the enqueueing span instead - see ``db.write.queue_wait`` above.",
-    (ISOLATED_CONNECTION, TRANSACTION),
-)
-
-STARTUP = SpanName(
-    "datasette.startup",
-    "``invoke_startup()`` running: ``register_events``, ``register_actions``, "
-    "``register_column_types``, ``prepare_jinja2_environment``, internal-database "
-    "schema catalog refresh (including the ``prepare_connection`` warm-up this "
-    "triggers for each database touched for the first time), saved queries, "
-    "column type config and the ``startup`` hook. Runs once per process, before "
-    "any request exists, so without this span every child it creates would be "
-    "its own orphan root trace. A connection warmed later - lazily, the first "
-    "time a *request* touches a new database or thread - nests under that "
-    "request's own span instead, not under this one, since this span has "
-    "already ended by then.",
-)
-
-SPANS = (
-    HTTP_REQUEST,
-    DB_QUERY,
-    DB_QUERY_EXECUTE,
-    DB_WRITE_QUEUE_WAIT,
-    DB_WRITE_EXECUTE,
-    STARTUP,
-)
-
-
-def span_for(emitted_name, kind=None):
-    """
-    Resolve an emitted span name to its registry entry, or None.
-
-    Handles the two entry kinds whose emitted names are not knowable in
-    advance:
-
-    - `prefix=True` - the name carries a variable suffix, matched by prefix.
-      Phase 1 registers none.
-    - `dynamic=True` - the name has no fixed part at all, so it is matched on
-      `kind` instead and the caller has to supply one. Exact and prefix
-      entries are tried first, so a dynamic entry can never shadow a span
-      that does have a registered name.
-    """
-    for span in SPANS:
-        if span.dynamic:
-            continue
-        if span.prefix:
-            if emitted_name.startswith(span):
-                return span
-        elif emitted_name == span:
-            return span
-    if kind is not None:
-        for span in SPANS:
-            if span.dynamic and span.kind == kind:
-                return span
-    return None
-
-
-def attribute_allowed(span, emitted_key):
-    "Whether `emitted_key` is a registered attribute of `span`."
-    if span is None:
-        return False
-    return emitted_key in span.attributes

datasette/templates/api_explorer.html

@@ -3,7 +3,6 @@
 {% block title %}API Explorer{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% endblock %}
 
 {% block content %}
@@ -126,7 +125,7 @@ getForm.addEventListener("submit", (ev) => {
     document.getElementById('response-status').textContent = response.status;
     return response.json();
   }).then((data) => {
-    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
+    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
     errorList.style.display = 'none';
   }).catch((error) => {
     alert(error);
@@ -174,7 +173,7 @@ postForm.addEventListener("submit", (ev) => {
     } else {
       errorList.style.display = 'none';
     }
-    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
+    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
     output.style.display = 'block';
   }).catch(err => {
     alert("Error: " + err);

datasette/templates/debug_allowed.html

@@ -3,7 +3,6 @@
 {% block title %}Allowed Resources{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -198,7 +197,7 @@ function displayResults(data) {
     }
 
     // Update raw JSON
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayError(data) {
@@ -208,7 +207,7 @@ function displayError(data) {
 
     resultsContent.innerHTML = `<div class="error-message">Error: ${escapeHtml(data.error || 'Unknown error')}</div>`;
 
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 // Disable child input if parent is empty

datasette/templates/debug_check.html

@@ -3,7 +3,6 @@
 {% block title %}Explain a permission decision{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 <style>
@@ -238,7 +237,7 @@ function displayResult(data) {
     displayRules(data.explanation);
     displayRestrictions(data.explanation.restrictions);
     displayRequirements(data.explanation.required_actions);
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayRules(explanation) {
@@ -298,7 +297,7 @@ function displayError(data) {
     document.getElementById('matching-rules').innerHTML = '';
     document.getElementById('restrictions-section').style.display = 'none';
     document.getElementById('requirements-section').style.display = 'none';
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 form.addEventListener('submit', event => {

datasette/templates/debug_rules.html

@@ -3,7 +3,6 @@
 {% block title %}Permission Rules{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -185,7 +184,7 @@ function displayResults(data) {
     }
 
     // Update raw JSON
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayError(data) {
@@ -195,7 +194,7 @@ function displayError(data) {
 
     resultsContent.innerHTML = `<div class="error-message">Error: ${escapeHtml(data.error || 'Unknown error')}</div>`;
 
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 </script>

datasette/tracer.py

@@ -0,0 +1,156 @@
+import asyncio
+import json
+import time
+import traceback
+from contextlib import contextmanager
+from contextvars import ContextVar
+
+from markupsafe import escape
+
+tracers = {}
+
+TRACE_RESERVED_KEYS = {"type", "start", "end", "duration_ms", "traceback"}
+
+trace_task_id = ContextVar("trace_task_id", default=None)
+
+
+def get_task_id():
+    current = trace_task_id.get(None)
+    if current is not None:
+        return current
+    try:
+        loop = asyncio.get_event_loop()
+    except RuntimeError:
+        return None
+    return id(asyncio.current_task(loop=loop))
+
+
+@contextmanager
+def trace_child_tasks():
+    token = trace_task_id.set(get_task_id())
+    try:
+        yield
+    finally:
+        trace_task_id.reset(token)
+
+
+@contextmanager
+def trace(trace_type, **kwargs):
+    assert not TRACE_RESERVED_KEYS.intersection(
+        kwargs.keys()
+    ), f".trace() keyword parameters cannot include {TRACE_RESERVED_KEYS}"
+    task_id = get_task_id()
+    if task_id is None:
+        yield kwargs
+        return
+    tracer = tracers.get(task_id)
+    if tracer is None:
+        yield kwargs
+        return
+    start = time.perf_counter()
+    captured_error = None
+    try:
+        yield kwargs
+    except Exception as ex:
+        captured_error = ex
+        raise
+    finally:
+        end = time.perf_counter()
+        trace_info = {
+            "type": trace_type,
+            "start": start,
+            "end": end,
+            "duration_ms": (end - start) * 1000,
+            "traceback": traceback.format_list(traceback.extract_stack(limit=6)[:-3]),
+            "error": str(captured_error) if captured_error else None,
+        }
+        trace_info.update(kwargs)
+        tracer.append(trace_info)
+
+
+@contextmanager
+def capture_traces(tracer):
+    # tracer is a list
+    task_id = get_task_id()
+    if task_id is None:
+        yield
+        return
+    tracers[task_id] = tracer
+    yield
+    del tracers[task_id]
+
+
+class AsgiTracer:
+    # If the body is larger than this we don't attempt to append the trace
+    max_body_bytes = 1024 * 256  # 256 KB
+
+    def __init__(self, app):
+        self.app = app
+
+    async def __call__(self, scope, receive, send):
+        if b"_trace=1" not in scope.get("query_string", b"").split(b"&"):
+            await self.app(scope, receive, send)
+            return
+        trace_start = time.perf_counter()
+        traces = []
+
+        accumulated_body = b""
+        size_limit_exceeded = False
+        response_headers = []
+
+        async def wrapped_send(message):
+            nonlocal accumulated_body, size_limit_exceeded, response_headers
+
+            if message["type"] == "http.response.start":
+                response_headers = message["headers"]
+                await send(message)
+                return
+
+            if message["type"] != "http.response.body" or size_limit_exceeded:
+                await send(message)
+                return
+
+            # Accumulate body until the end or until size is exceeded
+            accumulated_body += message["body"]
+            if len(accumulated_body) > self.max_body_bytes:
+                # Send what we have accumulated so far
+                await send(
+                    {
+                        "type": "http.response.body",
+                        "body": accumulated_body,
+                        "more_body": bool(message.get("more_body")),
+                    }
+                )
+                size_limit_exceeded = True
+                return
+
+            if not message.get("more_body"):
+                # We have all the body - modify it and send the result
+                # TODO: What to do about Content-Type or other cases?
+                trace_info = {
+                    "request_duration_ms": 1000 * (time.perf_counter() - trace_start),
+                    "sum_trace_duration_ms": sum(t["duration_ms"] for t in traces),
+                    "num_traces": len(traces),
+                    "traces": traces,
+                }
+                content_type = next(
+                    (
+                        v.decode("utf8")
+                        for k, v in response_headers
+                        if k.lower() == b"content-type"
+                    ),
+                    "",
+                )
+                if "text/html" in content_type and b"</body>" in accumulated_body:
+                    extra = escape(json.dumps(trace_info, indent=2))
+                    extra_html = f"<pre>{extra}</pre></body>".encode()
+                    accumulated_body = accumulated_body.replace(b"</body>", extra_html)
+                elif "json" in content_type and accumulated_body.startswith(b"{"):
+                    data = json.loads(accumulated_body.decode("utf8"))
+                    if "_trace" not in data:
+                        data["_trace"] = trace_info
+                    accumulated_body = json.dumps(data).encode("utf8")
+                await send({"type": "http.response.body", "body": accumulated_body})
+
+        with capture_traces(traces):
+            await self.app(scope, receive, wrapped_send)

datasette/utils/__init__.py

@@ -820,7 +820,8 @@ def detect_spatialite(conn):
 
 def detect_fts(conn, table):
     """Detect if table has a corresponding FTS virtual table and return it"""
-    rows = conn.execute(detect_fts_sql(table)).fetchall()
+    sql, params = detect_fts_sql(table)
+    rows = conn.execute(sql, params).fetchall()
     if len(rows) == 0:
         return None
     else:
@@ -828,18 +829,26 @@ def detect_fts(conn, table):
 
 
 def detect_fts_sql(table):
-    return r"""
-        select name from sqlite_master
-            where rootpage = 0
-            and (
-                sql like '%VIRTUAL TABLE%USING FTS%content="{table}"%'
-                or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%'
-                or (
-                    tbl_name = "{table}"
-                    and sql like '%VIRTUAL TABLE%USING FTS%'
+    escaped_table = table.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+    return (
+        r"""
+            select name from sqlite_master
+                where rootpage = 0
+                and (
+                    sql like :fts_double_quoted escape char(92)
+                    or sql like :fts_bracket_quoted escape char(92)
+                    or (
+                        tbl_name = :table
+                        and sql like '%VIRTUAL TABLE%USING FTS%'
+                    )
                 )
-            )
-    """.format(table=table.replace("'", "''"))
+        """,
+        {
+            "fts_double_quoted": f'%VIRTUAL TABLE%USING FTS%content="{escaped_table}"%',
+            "fts_bracket_quoted": f"%VIRTUAL TABLE%USING FTS%content=[{escaped_table}]%",
+            "table": table,
+        },
+    )
 
 
 def detect_json1(conn=None):
@@ -1557,7 +1566,13 @@ async def row_sql_params_pks(db, table, pk_values):
     if use_rowid:
         select = "rowid, *"
         pks = ["rowid"]
-    wheres = [f'"{pk}"=:p{i}' for i, pk in enumerate(pks)]
+    wheres = []
+    for i, pk in enumerate(pks):
+        escaped_pk = escape_sqlite(pk)
+        # Preserve the historic always-quoted SQL exposed by _extra=query
+        if escaped_pk == pk:
+            escaped_pk = f'"{pk}"'
+        wheres.append(f"{escaped_pk}=:p{i}")
     sql = f"select {select} from {escape_sqlite(table)} where {' AND '.join(wheres)}"
     params = {}
     for i, pk_value in enumerate(pk_values):
@@ -1729,7 +1744,7 @@ def redact_keys(original: dict, key_patterns: Iterable) -> dict:
             return {
                 k: (
                     redact(v)
-                    if not any(pattern in k for pattern in key_patterns)
+                    if not any(pattern in k.casefold() for pattern in key_patterns)
                     else "***"
                 )
                 for k, v in data.items()

datasette/utils/actions_sql.py

@@ -29,6 +29,15 @@ from datasette.utils.permissions import gather_permission_sql_from_hooks
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
+    from datasette.permissions import Action
+
+
+def _child_collation(action: "Action") -> str:
+    """Match resource identity without changing the spelling returned by SQL."""
+    resource_class = action.resource_class
+    if resource_class is not None and resource_class.case_insensitive_child:
+        return "NOCASE"
+    return "BINARY"
 
 
 async def build_allowed_resources_sql(
@@ -149,6 +158,7 @@ async def _build_single_action_sql(
         raise ValueError(f"Unknown action: {action}")
 
     # Get base resources SQL from the resource class
+    child_collation = _child_collation(action_obj)
     base_resources_sql = await action_obj.resource_class.resources_sql(
         datasette, actor=actor
     )
@@ -185,7 +195,7 @@ async def _build_single_action_sql(
         if permission_sql.sql is None:
             continue
         rule_sqls.append(f"""
-            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
+            SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
             """.strip())
@@ -299,9 +309,9 @@ async def _build_single_action_sql(
         query_parts.extend(
             ["anon_child_agg AS ("]
             + _anon_agg(
-                "parent, child,",
+                f"parent, child COLLATE {child_collation} AS child,",
                 "parent IS NOT NULL AND child IS NOT NULL",
-                "parent, child",
+                f"parent, child COLLATE {child_collation}",
             )
             + ["),", "anon_parent_agg AS ("]
             + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent")
@@ -382,7 +392,8 @@ async def _build_single_action_sql(
         # Wrap each restriction_sql in a subquery to avoid operator precedence issues
         # with UNION ALL inside the restriction SQL statements
         restriction_intersect = "\nINTERSECT\n".join(
-            f"SELECT * FROM ({sql})" for sql in restriction_sqls
+            f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})"
+            for sql in restriction_sqls
         )
         # Decompose by NULL-pattern so the final filter can use pure-equality
         # EXISTS lookups (satisfiable via automatic indexes) instead of a
@@ -480,6 +491,7 @@ async def build_permission_rules_sql(
     union_parts = []
     all_params = {}
     restriction_sqls = []
+    child_collation = _child_collation(action_obj)
 
     for permission_sql in permission_sqls:
         all_params.update(permission_sql.params or {})
@@ -493,7 +505,7 @@ async def build_permission_rules_sql(
             continue
 
         union_parts.append(f"""
-            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
+            SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
             """.strip())
@@ -564,6 +576,7 @@ async def check_permissions_for_actions(
     verdicts = {}
 
     for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)):
+        child_collation = _child_collation(datasette.actions[action])
         prefix = f"a{i}_"
         rule_parts = []
         restriction_parts = []
@@ -589,7 +602,7 @@ async def check_permissions_for_actions(
             if sql is None:
                 continue
             rule_parts.append(
-                f"SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
+                f"SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
             )
 
         if not rule_parts:
@@ -623,7 +636,8 @@ async def check_permissions_for_actions(
         if restriction_parts:
             # Database-level restrictions (parent, NULL) match all children
             restriction_intersect = "\nINTERSECT\n".join(
-                f"SELECT * FROM ({sql})" for sql in restriction_parts
+                f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})"
+                for sql in restriction_parts
             )
             ctes.append(f"a{i}_restriction AS (\n{restriction_intersect}\n)")
             verdict_sql = f"""({verdict_sql}) AND EXISTS (
@@ -770,6 +784,7 @@ async def _explain_single_action(
     db = datasette.get_internal_database()
     matched_rules = []
     restrictions = []
+    child_collation = _child_collation(datasette.actions[action])
 
     for permission_sql in permission_sqls:
         params = dict(permission_sql.params or {})
@@ -784,7 +799,7 @@ async def _explain_single_action(
                 SELECT parent, child, allow, reason
                 FROM ({permission_sql.sql}) AS permission_rules
                 WHERE (parent IS NULL OR parent = :{parent_param})
-                  AND (child IS NULL OR child = :{child_param})
+                  AND (child IS NULL OR child COLLATE {child_collation} = :{child_param})
                 """,
                 params,
             )
@@ -811,7 +826,7 @@ async def _explain_single_action(
                     SELECT EXISTS(
                         SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules
                         WHERE (parent IS NULL OR parent = :{parent_param})
-                          AND (child IS NULL OR child = :{child_param})
+                          AND (child IS NULL OR child COLLATE {child_collation} = :{child_param})
                     ) AS resource_is_in_allowlist
                     """,
                     params,

datasette/utils/asgi.py

@@ -498,6 +498,8 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None):
             await asgi_send_html(send, "404: File not found", 404)
             return
 
+    # Only the actual static-file handler can bypass dynamic response privacy.
+    inner_static._datasette_static = True
     return inner_static
 
 

datasette/utils/sql_analysis.py

@@ -1,6 +1,8 @@
+import sys
 from dataclasses import dataclass
 from typing import Literal
 
+from datasette.utils import escape_sqlite
 from datasette.utils.sqlite import SQLiteTableType, sqlite3, sqlite_table_type
 
 SQLOperation = Literal[
@@ -195,6 +197,16 @@ def _allow_authorizer_action(*args):
     return sqlite3.SQLITE_OK
 
 
+def _disable_authorizer(conn):
+    # Python 3.11 added support for unregistering an authorizer using None.
+    # On Python 3.10, None is installed as the callback instead, and the next
+    # statement fails with "not authorized" when sqlite3 tries to call it.
+    if sys.version_info >= (3, 11):
+        conn.set_authorizer(None)
+    else:
+        conn.set_authorizer(_allow_authorizer_action)
+
+
 def analyze_sql_tables(
     conn,
     sql: str,
@@ -208,7 +220,9 @@ def analyze_sql_tables(
 
     This function is synchronous and connection-based. It temporarily installs a
     SQLite authorizer, prepares ``EXPLAIN <sql>``, and returns the operation
-    callbacks observed while SQLite compiles the statement.
+    callbacks observed while SQLite compiles the statement. ``CREATE VIEW`` is
+    additionally executed inside a rolled-back savepoint so its source-table reads
+    can be discovered by analyzing a query against the temporary view.
     """
     operations: dict[OperationKey, set[str]] = {}
 
@@ -481,7 +495,7 @@ def analyze_sql_tables(
                         conn, key.table, schema=key.sqlite_schema
                     )
     finally:
-        conn.set_authorizer(None)
+        _disable_authorizer(conn)
 
     has_schema_operation = any(
         key.target_type in {"table", "index", "view", "trigger", "virtual-table"}
@@ -532,7 +546,7 @@ def analyze_sql_tables(
             return None
         return table_kind_cache[(key.sqlite_schema, key.table)]
 
-    return SQLAnalysis(
+    analysis = SQLAnalysis(
         operations=tuple(
             Operation(
                 operation=key.operation,
@@ -549,3 +563,58 @@ def analyze_sql_tables(
             for key, columns in operations.items()
         )
     )
+
+    # SQLite does not resolve the SELECT body of a view when preparing CREATE
+    # VIEW, so its authorizer does not report reads from the view's source
+    # tables. Temporarily create the view, analyze a query against it (which
+    # does resolve the body), then roll the schema change back. Database-level
+    # callers use an isolated writable connection for this analysis.
+    create_view_operations = tuple(
+        operation
+        for operation in analysis.operations
+        if operation.operation == "create" and operation.target_type == "view"
+    )
+    if not create_view_operations:
+        return analysis
+
+    savepoint = "datasette_analyze_create_view"
+    conn.execute(f"SAVEPOINT {savepoint}")
+    try:
+        conn.execute(sql, params if params is not None else {})
+        dependency_reads = []
+        for view_operation in create_view_operations:
+            if view_operation.sqlite_schema is None or view_operation.table is None:
+                raise sqlite3.OperationalError(
+                    "Could not determine the created view name"
+                )
+            quoted_schema = escape_sqlite(view_operation.sqlite_schema)
+            quoted_view = escape_sqlite(view_operation.table)
+            qualified_view = f"{quoted_schema}.{quoted_view}"
+            view_analysis = analyze_sql_tables(
+                conn,
+                f"SELECT * FROM {qualified_view}",
+                database_name=database_name,
+                schema_to_database=schema_to_database,
+            )
+            dependency_reads.extend(
+                operation
+                for operation in view_analysis.operations
+                if operation.operation == "read"
+                and not (
+                    operation.sqlite_schema == view_operation.sqlite_schema
+                    and operation.table == view_operation.table
+                )
+            )
+    finally:
+        conn.execute(f"ROLLBACK TO {savepoint}")
+        conn.execute(f"RELEASE {savepoint}")
+
+    existing_operations = set(analysis.operations)
+    return SQLAnalysis(
+        operations=analysis.operations
+        + tuple(
+            operation
+            for operation in dependency_reads
+            if operation not in existing_operations
+        )
+    )

datasette/utils/sqlite.py

@@ -15,8 +15,17 @@ if hasattr(sqlite3, "enable_callback_tracebacks"):
 _cached_sqlite_version = None
 _cached_supports_returning = None
 SQLiteTableType = Literal["table", "view", "virtual", "shadow"]
+_SQLITE_IDENTIFIER_RE = (
+    r"""(?:"(?:[^"]|"")*"|'(?:[^']|'')*'|`(?:[^`]|``)*`|\[[^\]]*\]|[^\s.()'"`\[\]]+)"""
+)
 _VIRTUAL_TABLE_MODULE_RE = re.compile(
-    r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)",
+    r"^\s*CREATE\s+VIRTUAL\s+TABLE\b\s*(?:IF\s+NOT\s+EXISTS\s+)?"
+    + _SQLITE_IDENTIFIER_RE
+    + r"(?:\s*\.\s*"
+    + _SQLITE_IDENTIFIER_RE
+    + r")?\s*\bUSING\b\s*("
+    + _SQLITE_IDENTIFIER_RE
+    + r")",
     re.IGNORECASE | re.DOTALL,
 )
 _VIRTUAL_TABLE_SHADOW_SUFFIXES = {
@@ -83,19 +92,53 @@ def sqlite_table_type(
 ) -> SQLiteTableType | None:
     if supports_table_list():
         try:
-            query = "select type from pragma_table_list where name = ?"
-            params: tuple[str, ...] = (table,)
+            # Use the "PRAGMA table_list" statement form rather than the
+            # pragma_table_list(...) table-valued function. The
+            # table-valued function is resolved like an ordinary relation
+            # name, so an attacker-created table or view literally named
+            # "pragma_table_list" can shadow it and spoof the reported
+            # type (e.g. claiming a virtual table is an ordinary table).
+            # The PRAGMA statement form is a distinct piece of SQL syntax
+            # that always invokes SQLite's built-in pragma, so it cannot
+            # be shadowed by a user-created relation.
             if schema is not None:
-                query += " and schema = ?"
-                params = (table, schema)
-            row = conn.execute(query, params).fetchone()
-            if row is not None and row[0] in {"table", "view", "virtual", "shadow"}:
-                return row[0]
+                query = f"PRAGMA {_quote_identifier(schema)}.table_list"
+            else:
+                query = "PRAGMA table_list"
+            cursor = conn.execute(query)
+            columns = [description[0] for description in cursor.description]
+            for row in cursor.fetchall():
+                record = dict(zip(columns, row))
+                if record.get("name") != table:
+                    continue
+                if schema is not None and record.get("schema") != schema:
+                    continue
+                row_type = record.get("type")
+                if row_type in {"table", "view", "virtual", "shadow"}:
+                    return row_type
         except sqlite3.DatabaseError:
             pass
     return _sqlite_table_type_from_schema(conn, table, schema=schema)
 
 
+def check_structured_write_table(conn, table: str, *, allow_missing=False):
+    """Validate a row-write target on the connection that will perform the write."""
+    # SQLite resolves identifiers case-insensitively. The create API must not
+    # treat a differently cased existing name as a missing table.
+    row = conn.execute(
+        "select name from main.sqlite_master where name = ? collate nocase "
+        "and type in ('table', 'view')",
+        (table,),
+    ).fetchone()
+    if row is None and allow_missing:
+        return
+    if row is not None and sqlite_table_type(conn, row[0]) == "table":
+        return
+    # Virtual table modules can interpret row writes as administrative operations.
+    # Their shadow tables are internal storage, not independently writable data.
+    raise ValueError("Structured writes require an ordinary table")
+
+
 def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]:
     schema_table = _sqlite_schema_table(schema)
     try:
@@ -118,6 +161,63 @@ def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]
     return sorted(hidden_tables) + content_fts_tables
 
 
+def sqlite_derived_table_dependencies(
+    conn, *, schema: str | None = "main"
+) -> dict[str, str]:
+    """Return implementation table -> logical/content table dependencies.
+
+    ``PRAGMA table_list`` safely identifies virtual and shadow tables, but
+    does not report which virtual table owns a shadow table or which table is
+    named by an FTS ``content=`` option. Derive those relationships from
+    ``sqlite_master`` DDL and the documented shadow-table suffixes.
+
+    Database errors propagate: failed discovery must not be mistaken for an
+    empty dependency map and cached as permission to skip inheritance.
+    """
+    schema_table = _sqlite_schema_table(schema)
+    rows = conn.execute(
+        f"select name, sql from {schema_table} where type = 'table'"
+    ).fetchall()
+
+    table_names = {row[0] for row in rows}
+    # SQLite identifiers fold ASCII letters only.
+    identifier_case = str.maketrans(
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
+    )
+    canonical_names = {name.translate(identifier_case): name for name in table_names}
+    dependencies = {}
+    for virtual_table, sql in rows:
+        module = _virtual_table_module(sql)
+        if module is None:
+            continue
+
+        # SQLite's documented shadow tables are implementation details of
+        # their logical virtual table.
+        for suffix in _VIRTUAL_TABLE_SHADOW_SUFFIXES.get(module, ()):
+            shadow_table = virtual_table + suffix
+            if shadow_table in table_names:
+                dependencies[shadow_table] = virtual_table
+
+        # An external-content FTS table can expose values fetched from its
+        # content table, so it must also depend on that table's permission.
+        if module in {"fts3", "fts4", "fts5"}:
+            content_table = _fts_external_content_table(sql)
+            if content_table:
+                dependencies[virtual_table] = content_table
+
+        if module in {"fts5vocab", "fts4aux"}:
+            source = _fts_vocabulary_source(sql, module, schema or "main")
+            source = (
+                canonical_names.get(source.translate(identifier_case))
+                if source
+                else None
+            )
+            # An unresolved source is itself derived, so the one-hop policy denies it.
+            dependencies[virtual_table] = source or virtual_table
+
+    return dependencies
+
+
 def _sqlite_table_type_from_schema(
     conn,
     table: str,
@@ -184,10 +284,151 @@ def _quote_identifier(value: str) -> str:
 def _virtual_table_module(sql: str | None) -> str | None:
     if not sql:
         return None
+    match = _VIRTUAL_TABLE_MODULE_RE.search(_strip_sql_comments(sql))
+    if match is None:
+        return None
+    return _unquote_sql_value(match.group(1)).lower()
+
+
+def _fts_external_content_table(sql: str | None) -> str | None:
+    """Extract the external ``content=`` table from an FTS declaration."""
+    if not sql:
+        return None
+    sql = _strip_sql_comments(sql)
     match = _VIRTUAL_TABLE_MODULE_RE.search(sql)
     if match is None:
         return None
-    return match.group(1).strip("\"'[]`").lower()
+    open_paren = sql.find("(", match.end())
+    if open_paren == -1:
+        return None
+    close_paren = sql.rfind(")")
+    if close_paren <= open_paren:
+        return None
+
+    for argument in _split_sql_arguments(sql[open_paren + 1 : close_paren]):
+        key, separator, value = argument.partition("=")
+        if not separator or key.strip().lower() != "content":
+            continue
+        return _unquote_sql_value(value.strip())
+    return None
+
+
+def _fts_vocabulary_source(sql: str, module: str, schema: str) -> str | None:
+    """Resolve a vocabulary source within the current SQLite schema.
+
+    Cross-schema sources cannot be represented by the dependency map and
+    are conservatively left unresolved.
+    """
+    sql = _strip_sql_comments(sql)
+    match = _VIRTUAL_TABLE_MODULE_RE.search(sql)
+    if match is None:
+        return None
+    start = sql.find("(", match.end())
+    end = sql.rfind(")")
+    if start < 0 or end <= start:
+        return None
+    arguments = [
+        _unquote_sql_value(arg.strip())
+        for arg in _split_sql_arguments(sql[start + 1 : end])
+    ]
+    expected = 2 if module == "fts5vocab" else 1
+    if len(arguments) == expected:
+        return arguments[0]
+    if len(arguments) == expected + 1 and arguments[0].lower() == schema.lower():
+        return arguments[1]
+    return None
+
+
+def _split_sql_arguments(arguments: str) -> list[str]:
+    """Split comma-separated SQLite arguments without splitting quoted text."""
+    parts = []
+    start = 0
+    quote = None
+    closing_quote = None
+    index = 0
+    while index < len(arguments):
+        char = arguments[index]
+        if quote is None:
+            if char in {"'", '"', "`", "["}:
+                quote = char
+                closing_quote = "]" if char == "[" else char
+            elif char == ",":
+                parts.append(arguments[start:index])
+                start = index + 1
+        elif char == closing_quote:
+            # Single/double/backtick quoting escapes the delimiter by
+            # doubling it. Square-bracket identifiers do not.
+            if (
+                quote != "["
+                and index + 1 < len(arguments)
+                and arguments[index + 1] == closing_quote
+            ):
+                index += 1
+            else:
+                quote = None
+                closing_quote = None
+        index += 1
+    parts.append(arguments[start:])
+    return parts
+
+
+def _strip_sql_comments(sql: str) -> str:
+    """Remove SQLite comments while preserving quoted strings/identifiers."""
+    output = []
+    quote = None
+    closing_quote = None
+    index = 0
+    while index < len(sql):
+        char = sql[index]
+        next_char = sql[index + 1] if index + 1 < len(sql) else ""
+        if quote is None:
+            if char in {"'", '"', "`", "["}:
+                quote = char
+                closing_quote = "]" if char == "[" else char
+                output.append(char)
+            elif char == "-" and next_char == "-":
+                index += 2
+                while index < len(sql) and sql[index] not in "\r\n":
+                    index += 1
+                output.append(" ")
+                continue
+            elif char == "/" and next_char == "*":
+                index += 2
+                while index + 1 < len(sql) and sql[index : index + 2] != "*/":
+                    index += 1
+                index = min(index + 2, len(sql))
+                output.append(" ")
+                continue
+            else:
+                output.append(char)
+        else:
+            output.append(char)
+            if char == closing_quote:
+                if (
+                    quote != "["
+                    and index + 1 < len(sql)
+                    and sql[index + 1] == closing_quote
+                ):
+                    output.append(sql[index + 1])
+                    index += 1
+                else:
+                    quote = None
+                    closing_quote = None
+        index += 1
+    return "".join(output)
+
+
+def _unquote_sql_value(value: str) -> str:
+    if len(value) < 2:
+        return value
+    pairs = {"'": "'", '"': '"', "`": "`", "[": "]"}
+    closing = pairs.get(value[0])
+    if closing is None or value[-1] != closing:
+        return value
+    unquoted = value[1:-1]
+    if value[0] != "[":
+        unquoted = unquoted.replace(closing * 2, closing)
+    return unquoted
 
 
 def _is_fts_content_virtual_table(sql: str | None) -> bool:

datasette/utils/testing.py

@@ -4,7 +4,7 @@ from urllib.parse import urlencode
 from asgiref.sync import async_to_sync
 
 # These wrapper classes pre-date the introduction of
-# datasette.client and httpx to Datasette. They could
+# datasette.client and httpx2 to Datasette. They could
 # be removed if the Datasette tests are modified to
 # call datasette.client directly.
 

datasette/version.py

@@ -1,2 +1,2 @@
-__version__ = "1.0a38"
+__version__ = "1.0a39"
 __version_info__ = tuple(__version__.split("."))

datasette/views/base.py

@@ -3,6 +3,7 @@ import hashlib
 import sys
 
 from datasette.utils import (
+    EscapeHtmlWriter,
     InvalidSql,
     LimitedWriter,
     add_cors_headers,
@@ -224,11 +225,26 @@ async def stream_csv(datasette, fetch_data, request, database):
                 headings.append(f"{column}_label")
 
     content_type = "text/plain; charset=utf-8"
+    preamble = ""
+    postamble = ""
+
+    trace = request.args.get("_trace")
+    if trace:
+        content_type = "text/html; charset=utf-8"
+        preamble = (
+            "<html><head><title>CSV debug</title></head>"
+            '<body><textarea style="width: 90%; height: 70vh">'
+        )
+        postamble = "</textarea></body></html>"
 
     async def stream_fn(r):
-        nonlocal data
+        nonlocal data, trace
         limited_writer = LimitedWriter(r, datasette.setting("max_csv_mb"))
-        writer = csv.writer(limited_writer)
+        if trace:
+            await limited_writer.write(preamble)
+            writer = csv.writer(EscapeHtmlWriter(limited_writer))
+        else:
+            writer = csv.writer(limited_writer)
         first = True
         next = None
         while first or (next and stream):
@@ -306,12 +322,14 @@ async def stream_csv(datasette, fetch_data, request, database):
                 sys.stderr.flush()
                 await r.write(str(ex))
                 return
+        await limited_writer.write(postamble)
 
     headers = {}
     if datasette.cors:
         add_cors_headers(headers)
     if request.args.get("_dl", None):
-        content_type = "text/csv; charset=utf-8"
+        if not trace:
+            content_type = "text/csv; charset=utf-8"
         disposition = 'attachment; filename="{}.csv"'.format(
             request.url_vars.get("table", database)
         )

datasette/views/database.py

@@ -40,7 +40,11 @@ from datasette.write_sql import QueryWriteRejected
 
 from . import Context
 from .base import DatasetteError, View, stream_csv
-from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns
+from .query_helpers import (
+    _block_framing,
+    _ensure_stored_query_execution_permissions,
+    _table_columns,
+)
 from .table_create_alter import _create_table_ui_context
 from .table_extras import (
     QueryExtraContext,
@@ -857,7 +861,8 @@ class QueryView(View):
                 raise DatasetteError("?sql= is required", status=400)
 
             async def fetch_data_for_csv(request, _next=None):
-                results = await db.execute(sql, params, truncate=True)
+                # Reuse the trusted magic parameter values prepared above.
+                results = await db.execute(sql, params_for_query, truncate=True)
                 data = {"rows": results.rows, "columns": results.columns}
                 return data, None, None
 
@@ -1140,6 +1145,8 @@ class QueryView(View):
             assert False, f"Invalid format: {format_}"
         if datasette.cors:
             add_cors_headers(r.headers)
+        if stored_query_write and format_ == "html":
+            _block_framing(r)
         return r
 
 

datasette/views/execute_write.py

@@ -1,6 +1,7 @@
 import re
 from urllib.parse import urlencode
 
+from datasette.database import QueryInterrupted
 from datasette.resources import DatabaseResource
 from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3
 from datasette.utils.asgi import Response
@@ -384,7 +385,7 @@ class ExecuteWriteView(BaseView):
         try:
             execute_write_kwargs = {"request": request}
             cursor = await db.execute_write(sql, params, **execute_write_kwargs)
-        except sqlite3.DatabaseError as ex:
+        except (QueryInterrupted, sqlite3.DatabaseError) as ex:
             message = str(ex)
             if wants_json:
                 return _block_framing(Response.error([message], 400))

datasette/views/row.py

@@ -28,9 +28,11 @@ from datasette.utils import (
     path_with_format,
     path_with_removed_args,
     sqlite3,
+    tilde_decode,
     to_css_class,
 )
 from datasette.utils.asgi import Forbidden, NotFound, PayloadTooLarge, Response
+from datasette.utils.sqlite import check_structured_write_table
 
 from . import Context, from_extra
 from .base import BaseView, DatasetteError, stream_csv
@@ -137,6 +139,12 @@ class RowContext(Context):
     )
 
 
+async def _database_and_table_resource_from_request(datasette, request):
+    db = await datasette.resolve_database(request)
+    table = tilde_decode(request.url_vars["table"])
+    return db, table, TableResource(database=db.name, table=table)
+
+
 class RowView(BaseView):
     name = "row"
 
@@ -263,7 +271,7 @@ class RowView(BaseView):
         if ttl is None or not ttl.isdigit():
             ttl = self.ds.setting("default_cache_ttl")
 
-        return self.set_response_headers(response, ttl)
+        return self.set_response_headers(response, ttl, request)
 
     async def html(self, request, data, extra_template_data, templates):
         extras = {}
@@ -376,38 +384,52 @@ class RowView(BaseView):
             },
         )
 
-    def set_response_headers(self, response, ttl):
+    def set_response_headers(self, response, ttl, request=None):
+        private = getattr(request, "_datasette_private_response", False)
         # Set far-future cache expiry
         if self.ds.cache_headers and response.status == 200:
-            ttl = int(ttl)
-            if ttl == 0:
-                ttl_header = "no-cache"
+            if private:
+                # This response is only visible to the current actor (denied
+                # to anonymous requests), so it must never be stored by a
+                # shared cache/CDN - and ?_ttl= must not override that.
+                response.headers["Cache-Control"] = "private, no-store"
+                response.headers["Vary"] = "Cookie"
             else:
-                ttl_header = f"max-age={ttl}"
-            response.headers["Cache-Control"] = ttl_header
+                ttl = int(ttl)
+                if ttl == 0:
+                    ttl_header = "no-cache"
+                else:
+                    ttl_header = f"max-age={ttl}"
+                response.headers["Cache-Control"] = ttl_header
         response.headers["Referrer-Policy"] = "no-referrer"
         if self.ds.cors:
             add_cors_headers(response.headers)
         return response
 
     async def data(self, request, default_labels=False):
-        resolved = await self.ds.resolve_row(request)
-        db = resolved.db
+        db, table, resource = await _database_and_table_resource_from_request(
+            self.ds, request
+        )
         database = db.name
-        table = resolved.table
-        pk_values = resolved.pk_values
 
-        # Ensure user has permission to view this row
+        # Check the URL resource before resolving the row, so a denied request
+        # cannot distinguish an existing primary key from a missing one.
         visible, private = await self.ds.check_visibility(
             request.actor,
             action="view-table",
-            resource=TableResource(database=database, table=table),
+            resource=resource,
         )
         if not visible:
             raise Forbidden("You do not have permission to view this table")
+        # Record whether this response is private (visible to this actor
+        # only) so set_response_headers() can set appropriate Cache-Control
+        # headers, regardless of which output format ends up being rendered.
+        request._datasette_private_response = private
 
+        resolved = await self.ds.resolve_row(request)
+        pk_values = resolved.pk_values
         results = await resolved.db.execute(
-            resolved.sql, resolved.params, truncate=True, table=table
+            resolved.sql, resolved.params, truncate=True
         )
         columns = [r[0] for r in results.description]
         rows = list(results.rows)
@@ -482,8 +504,8 @@ class RowView(BaseView):
             for row in display_rows:
                 for cell in row:
                     if cell["column"] in pk_set:
-                        cell["value"] = markupsafe.Markup(
-                            "<strong>{}</strong>".format(cell["value"])
+                        cell["value"] = markupsafe.Markup("<strong>{}</strong>").format(
+                            cell["value"]
                         )
 
             label_column = await db.label_column_for_table(table) if is_table else None
@@ -556,7 +578,7 @@ class RowView(BaseView):
                 "private": private,
                 "columns": reordered_columns,
                 "foreign_key_tables": await self.foreign_key_tables(
-                    database, table, pk_values
+                    database, table, pk_values, actor=request.actor
                 ),
                 "database_color": db.color,
                 "display_columns": display_columns,
@@ -633,12 +655,23 @@ class RowView(BaseView):
             ),
         )
 
-    async def foreign_key_tables(self, database, table, pk_values):
+    async def foreign_key_tables(self, database, table, pk_values, *, actor):
         if len(pk_values) != 1:
             return []
         db = self.ds.databases[database]
         all_foreign_keys = await db.get_all_foreign_keys()
-        foreign_keys = all_foreign_keys[table]["incoming"]
+        foreign_keys = []
+        table_permissions = {}
+        for fk in all_foreign_keys[table]["incoming"]:
+            other_table = fk["other_table"]
+            if other_table not in table_permissions:
+                table_permissions[other_table] = await self.ds.allowed(
+                    action="view-table",
+                    resource=TableResource(database=database, table=other_table),
+                    actor=actor,
+                )
+            if table_permissions[other_table]:
+                foreign_keys.append(fk)
         if len(foreign_keys) == 0:
             return []
 
@@ -652,9 +685,6 @@ class RowView(BaseView):
             ]
         )
         try:
-            # No table= here: this counts incoming references across every
-            # foreign key pointing at this row, so it spans many tables and
-            # there is no single value db.collection.name could take.
             rows = list(await db.execute(sql, {"id": pk_values[0]}))
         except QueryInterrupted:
             # Almost certainly hit the timeout
@@ -698,9 +728,24 @@ def _truncated_row_flash_label(label):
     return label[: ROW_FLASH_LABEL_MAX_LENGTH - 1] + "\u2026"
 
 
-async def _row_flash_message(db, action, resolved, row=None):
+async def _row_flash_message(
+    datasette, request, action, resolved, row=None, *, refresh_row=False
+):
     pk_label = ", ".join(resolved.pk_values)
-    label_column = await db.label_column_for_table(resolved.table)
+    # Mutation permission does not grant access to stored row labels.
+    if not await datasette.allowed(
+        action="view-table",
+        resource=TableResource(database=resolved.db.name, table=resolved.table),
+        actor=request.actor,
+    ):
+        return f"{action} row {pk_label}"
+
+    if refresh_row and row is None:
+        results = await resolved.db.execute(
+            resolved.sql, resolved.params, truncate=True
+        )
+        row = results.first()
+    label_column = await resolved.db.label_column_for_table(resolved.table)
     label = row_label_from_label_column(row or resolved.row, label_column)
     if label:
         label = _truncated_row_flash_label(label)
@@ -713,22 +758,28 @@ async def _resolve_row_and_check_permission(datasette, request, permission):
     from datasette.app import DatabaseNotFound, RowNotFound, TableNotFound
 
     try:
-        resolved = await datasette.resolve_row(request)
+        _, _, resource = await _database_and_table_resource_from_request(
+            datasette, request
+        )
     except DatabaseNotFound as e:
         return False, Response.error([f"Database not found: {e.database_name}"], 404)
+
+    # Check the URL resource before resolving the row, so a denied request
+    # cannot distinguish an existing primary key from a missing one.
+    if not await datasette.allowed(
+        action=permission,
+        resource=resource,
+        actor=request.actor,
+    ):
+        return False, Response.error(["Permission denied"], 403)
+
+    try:
+        resolved = await datasette.resolve_row(request)
     except TableNotFound as e:
         return False, Response.error([f"Table not found: {e.table}"], 404)
     except RowNotFound as e:
         return False, Response.error([f"Record not found: {e.pk_values}"], 404)
 
-    # Ensure user has permission to delete this row
-    if not await datasette.allowed(
-        action=permission,
-        resource=TableResource(database=resolved.db.name, table=resolved.table),
-        actor=request.actor,
-    ):
-        return False, Response.error(["Permission denied"], 403)
-
     return True, resolved
 
 
@@ -747,6 +798,7 @@ class RowDeleteView(BaseView):
 
         # Delete table
         def delete_row(conn):
+            check_structured_write_table(conn, resolved.table)
             sqlite_utils.Database(conn)[resolved.table].delete(resolved.pk_values)
 
         try:
@@ -768,7 +820,7 @@ class RowDeleteView(BaseView):
             table_url = self.ds.urls.table(resolved.db.name, resolved.table)
             self.ds.add_message(
                 request,
-                await _row_flash_message(resolved.db, "Deleted", resolved),
+                await _row_flash_message(self.ds, request, "Deleted", resolved),
                 self.ds.INFO,
             )
             return Response.json({"ok": True, "redirect": str(table_url)}, status=200)
@@ -829,6 +881,7 @@ class RowUpdateView(BaseView):
             return Response.error(["Permission denied for alter-table"], 403)
 
         def update_row(conn):
+            check_structured_write_table(conn, resolved.table)
             sqlite_utils.Database(conn)[resolved.table].update(
                 resolved.pk_values, update, alter=alter
             )
@@ -841,9 +894,16 @@ class RowUpdateView(BaseView):
 
         result = {"ok": True}
         returned_row = None
-        if data.get("return"):
+        # Only read back and disclose the stored row if the actor is also
+        # allowed to view this table - update-row alone must not be usable
+        # to read data the actor cannot otherwise see.
+        if data.get("return") and await self.ds.allowed(
+            action="view-table",
+            resource=TableResource(database=resolved.db.name, table=resolved.table),
+            actor=request.actor,
+        ):
             results = await resolved.db.execute(
-                resolved.sql, resolved.params, truncate=True, table=resolved.table
+                resolved.sql, resolved.params, truncate=True
             )
             returned_row = results.dicts()[0]
             result["rows"] = [returned_row]
@@ -858,16 +918,15 @@ class RowUpdateView(BaseView):
         )
 
         if request.args.get("_message"):
-            message_row = returned_row
-            if message_row is None:
-                results = await resolved.db.execute(
-                    resolved.sql, resolved.params, truncate=True, table=resolved.table
-                )
-                message_row = results.first()
             self.ds.add_message(
                 request,
                 await _row_flash_message(
-                    resolved.db, "Updated", resolved, row=message_row
+                    self.ds,
+                    request,
+                    "Updated",
+                    resolved,
+                    row=returned_row,
+                    refresh_row=True,
                 ),
                 self.ds.INFO,
             )

datasette/views/special.py

@@ -311,6 +311,7 @@ class AllowedResourcesView(BaseView):
     has_json_alternate = False
 
     async def get(self, request):
+        await self.ds.ensure_permission(action="view-instance", actor=request.actor)
         await self.ds.refresh_schemas()
 
         # Check if user has permissions-debug (to show sensitive fields)
@@ -796,6 +797,8 @@ class CreateTokenView(BaseView):
             raise Forbidden(
                 "Token authentication cannot be used to create additional tokens"
             )
+        if "_r" in request.actor:
+            raise Forbidden("Restricted actors cannot create API tokens")
 
     async def shared(self, request):
         self.check_permission(request)
@@ -873,6 +876,11 @@ class CreateTokenView(BaseView):
                 else:
                     errors.append("Invalid expire duration unit")
 
+        if errors:
+            context = await self.shared(request)
+            context["errors"] = errors
+            return await self.render(["create_token.html"], request, context)
+
         # Are there any restrictions?
         from datasette.tokens import TokenRestrictions
 
@@ -1261,14 +1269,21 @@ class SchemaBaseView(BaseView):
 
     has_json_alternate = False
 
-    async def get_database_schema(self, database_name):
+    async def get_database_schema(self, database_name, actor):
         """Get schema SQL for a database."""
         db = self.ds.databases[database_name]
-        result = await db.execute(
-            "select group_concat(sql, ';' || CHAR(10)) as schema from sqlite_master where sql is not null"
+        allowed_tables_page = await self.ds.allowed_resources(
+            "view-table", actor, parent=database_name
+        )
+        allowed_table_names = {
+            resource.child async for resource in allowed_tables_page.all()
+        }
+        result = await db.execute(
+            "select tbl_name, sql from sqlite_master where sql is not null"
+        )
+        return ";\n".join(
+            row["sql"] for row in result.rows if row["tbl_name"] in allowed_table_names
         )
-        row = result.first()
-        return row["schema"] if row and row["schema"] else ""
 
     def format_json_response(self, data):
         """Format data as JSON response with CORS headers if needed."""
@@ -1330,7 +1345,7 @@ class InstanceSchemaView(SchemaBaseView):
         # Get schema for each database
         schemas = []
         for database_name in allowed_databases:
-            schema = await self.get_database_schema(database_name)
+            schema = await self.get_database_schema(database_name, request.actor)
             schemas.append({"database": database_name, "schema": schema})
 
         if format_ == "json":
@@ -1371,7 +1386,7 @@ class DatabaseSchemaView(SchemaBaseView):
         if database_name not in self.ds.databases:
             return self.format_error_response("Database not found", format_)
 
-        schema = await self.get_database_schema(database_name)
+        schema = await self.get_database_schema(database_name, request.actor)
 
         if format_ == "json":
             return self.format_json_response(
@@ -1410,7 +1425,8 @@ class TableSchemaView(SchemaBaseView):
         # Get schema for the table
         db = self.ds.databases[database_name]
         result = await db.execute(
-            "select sql from sqlite_master where name = ? and sql is not null",
+            "select sql from sqlite_master where name = ? "
+            "and type in ('table', 'view') and sql is not null",
             [table_name],
         )
         row = result.first()

datasette/views/stored_queries.py

@@ -279,7 +279,7 @@ class QueryCreateView(BaseView):
             ),
         )
         response.status = status
-        return response
+        return _block_framing(response)
 
     async def get(self, request):
         db = await self.ds.resolve_database(request)
@@ -527,7 +527,7 @@ class QueryEditView(BaseView):
             ),
         )
         response.status = status
-        return response
+        return _block_framing(response)
 
     async def get(self, request):
         db, query_name, existing = await self._load(request)
@@ -639,15 +639,17 @@ class QueryDeleteView(BaseView):
             return Response.error(
                 ["Trusted queries cannot be deleted using the API"], 403
             )
-        return await self.render(
-            ["query_delete.html"],
-            request,
-            {
-                "database": db.name,
-                "database_color": db.color,
-                "query": stored_query_to_dict(existing),
-                "query_url": self.ds.urls.table(db.name, query_name),
-            },
+        return _block_framing(
+            await self.render(
+                ["query_delete.html"],
+                request,
+                {
+                    "database": db.name,
+                    "database_color": db.color,
+                    "query": stored_query_to_dict(existing),
+                    "query_url": self.ds.urls.table(db.name, query_name),
+                },
+            )
         )
 
     async def post(self, request):

datasette/views/table.py

@@ -8,6 +8,7 @@ from dataclasses import dataclass, field
 import markupsafe
 import sqlite_utils
 
+from datasette import tracer
 from datasette.column_types import SQLiteType
 from datasette.database import QueryInterrupted
 from datasette.events import (
@@ -56,6 +57,7 @@ from datasette.utils.asgi import (
     Request,
     Response,
 )
+from datasette.utils.sqlite import check_structured_write_table
 
 from . import Context, from_extra
 from .base import BaseView, DatasetteError, stream_csv
@@ -1125,6 +1127,7 @@ class TableInsertView(BaseView):
             row_pk_values_for_later = [tuple(row[pk] for pk in pks) for row in rows]
 
         def insert_or_upsert_rows(conn):
+            check_structured_write_table(conn, table_name)
             table = sqlite_utils.Database(conn)[table_name]
             kwargs = {}
             if upsert:
@@ -1156,20 +1159,34 @@ class TableInsertView(BaseView):
             # TODO: narrow to expected write errors so Datasette bugs surface as 500s
             return Response.error([str(e)])
         result = {"ok": True}
+        # Only read back and disclose stored rows if the actor is also
+        # allowed to view this table - insert-row/update-row alone must
+        # not be usable to read data the actor cannot otherwise see.
+        if should_return and not await self.ds.allowed(
+            action="view-table",
+            resource=TableResource(database=database_name, table=table_name),
+            actor=request.actor,
+        ):
+            should_return = False
         if should_return:
             if upsert:
                 # Fetch based on initial input IDs
                 where_clause = " OR ".join(
-                    ["({})".format(" AND ".join(f"{pk} = ?" for pk in pks))]
+                    [
+                        "({})".format(
+                            " AND ".join(f"{escape_sqlite(pk)} = ?" for pk in pks)
+                        )
+                    ]
                     * len(row_pk_values_for_later)
                 )
                 args = list(itertools.chain.from_iterable(row_pk_values_for_later))
                 fetched_rows = await db.execute(
-                    "select {}* from [{}] where {}".format(
-                        "rowid, " if pks == ["rowid"] else "", table_name, where_clause
+                    "select {}* from {} where {}".format(
+                        "rowid, " if pks == ["rowid"] else "",
+                        escape_sqlite(table_name),
+                        where_clause,
                     ),
                     args,
-                    table=table_name,
                 )
                 result["rows"] = fetched_rows.dicts()
             else:
@@ -1383,7 +1400,7 @@ class TableDropView(BaseView):
                     "table": table_name,
                     "row_count": (
                         await db.execute(
-                            f"select count(*) from [{table_name}]", table=table_name
+                            f"select count(*) from {escape_sqlite(table_name)}"
                         )
                     ).single_value(),
                     "message": 'Pass "confirm": true to confirm',
@@ -1578,10 +1595,7 @@ class TableAutocompleteView(BaseView):
 
         try:
             results = await db.execute(
-                sql,
-                params,
-                custom_time_limit=AUTOCOMPLETE_TIME_LIMIT_MS,
-                table=table_name,
+                sql, params, custom_time_limit=AUTOCOMPLETE_TIME_LIMIT_MS
             )
         except QueryInterrupted:
             fallback_where = _autocomplete_prefix_like(pks[0])
@@ -1602,7 +1616,6 @@ class TableAutocompleteView(BaseView):
                     fallback_sql,
                     params,
                     custom_time_limit=AUTOCOMPLETE_TIME_LIMIT_MS,
-                    table=table_name,
                 )
             except QueryInterrupted:
                 return Response.json({"ok": True, "rows": []})
@@ -1689,7 +1702,8 @@ async def _sort_order(table_metadata, sortable_columns, request, order_by):
 
 async def table_view(datasette, request):
     await datasette.refresh_schemas()
-    response = await table_view_traced(datasette, request)
+    with tracer.trace_child_tasks():
+        response = await table_view_traced(datasette, request)
 
     # CORS
     if datasette.cors:
@@ -1700,13 +1714,22 @@ async def table_view(datasette, request):
     if ttl is None or not ttl.isdigit():
         ttl = datasette.setting("default_cache_ttl")
 
+    private = getattr(request, "_datasette_private_response", False)
+
     if datasette.cache_headers and response.status == 200:
-        ttl = int(ttl)
-        if ttl == 0:
-            ttl_header = "no-cache"
+        if private:
+            # This response is only visible to the current actor (denied to
+            # anonymous requests), so it must never be stored by a shared
+            # cache/CDN - and ?_ttl= must not be able to override that.
+            response.headers["Cache-Control"] = "private, no-store"
+            response.headers["Vary"] = "Cookie"
         else:
-            ttl_header = f"max-age={ttl}"
-        response.headers["Cache-Control"] = ttl_header
+            ttl = int(ttl)
+            if ttl == 0:
+                ttl_header = "no-cache"
+            else:
+                ttl_header = f"max-age={ttl}"
+            response.headers["Cache-Control"] = ttl_header
 
     # Referrer policy
     response.headers["Referrer-Policy"] = "no-referrer"
@@ -1954,6 +1977,10 @@ async def table_view_data(
     )
     if not visible:
         raise Forbidden("You do not have permission to view this table")
+    # Record whether this response is private (visible to this actor only)
+    # so the outer table_view() can set appropriate Cache-Control headers,
+    # regardless of which output format ends up being rendered.
+    request._datasette_private_response = private
 
     # Redirect based on request.args, if necessary
     redirect_response = await _redirect_if_needed(datasette, request, resolved)
@@ -2168,9 +2195,7 @@ async def table_view_data(
 
     # Execute the main query!
     try:
-        results = await db.execute(
-            sql, params, truncate=True, table=table_name, **extra_args
-        )
+        results = await db.execute(sql, params, truncate=True, **extra_args)
     except (sqlite3.OperationalError, InvalidSql) as e:
         raise DatasetteError(str(e), title="Invalid SQL", status=400)
 
@@ -2439,14 +2464,16 @@ async def _next_value_and_url(
             except IndexError:
                 # sort/sort_desc column missing from SELECT - look up value by PK instead
                 prefix_where_clause = " and ".join(
-                    f"[{pk}] = :pk{i}" for i, pk in enumerate(pks)
+                    f"{escape_sqlite(pk)} = :pk{i}" for i, pk in enumerate(pks)
+                )
+                prefix_lookup_sql = (
+                    f"select {escape_sqlite(sort or sort_desc)} "
+                    f"from {escape_sqlite(table_name)} where {prefix_where_clause}"
                 )
-                prefix_lookup_sql = f"select [{sort or sort_desc}] from [{table_name}] where {prefix_where_clause}"
                 prefix = (
                     await db.execute(
                         prefix_lookup_sql,
                         {**{f"pk{i}": rows[-2][pk] for i, pk in enumerate(pks)}},
-                        table=table_name,
                     )
                 ).single_value()
             if isinstance(prefix, dict) and "value" in prefix:

datasette/views/table_create_alter.py

@@ -27,7 +27,15 @@ from datasette.utils import (
     table_column_details,
 )
 from datasette.utils.asgi import NotFound, PayloadTooLarge, Response
-from datasette.utils.sqlite import sqlite_hidden_table_names
+from datasette.utils.permissions import (
+    SKIP_PERMISSION_CHECKS,
+    gather_permission_sql_from_hooks,
+    resolve_permissions_with_candidates,
+)
+from datasette.utils.sqlite import (
+    check_structured_write_table,
+    sqlite_hidden_table_names,
+)
 
 from .base import BaseView
 
@@ -122,6 +130,30 @@ def _public_foreign_key_target(target):
     }
 
 
+async def _filter_visible_foreign_key_targets(datasette, actor, database_name, targets):
+    if not targets:
+        return []
+
+    permission_sqls = await gather_permission_sql_from_hooks(
+        datasette=datasette,
+        actor=actor,
+        action="view-table",
+    )
+    if permission_sqls is SKIP_PERMISSION_CHECKS:
+        return targets
+
+    candidate_tables = list(dict.fromkeys(target["fk_table"] for target in targets))
+    permission_rows = await resolve_permissions_with_candidates(
+        datasette.get_internal_database(),
+        actor,
+        permission_sqls,
+        [(database_name, table_name) for table_name in candidate_tables],
+        "view-table",
+    )
+    visible_tables = {row["child"] for row in permission_rows if bool(row["allow"])}
+    return [target for target in targets if target["fk_table"] in visible_tables]
+
+
 def _singular(name):
     if name.endswith("ies") and len(name) > 3:
         return name[:-3] + "y"
@@ -821,16 +853,18 @@ class TableCreateView(BaseView):
         ignore = create_request.ignore
         replace = create_request.replace
 
+        table_name = create_request.table
+        table_exists = await db.table_exists(table_name)
+        table_resource = TableResource(database=database_name, table=table_name)
+
         # Replacing rows requires update-row permission
         if replace and not await self.ds.allowed(
             action="update-row",
-            resource=DatabaseResource(database=database_name),
+            resource=table_resource,
             actor=request.actor,
         ):
             return Response.error(["Permission denied: need update-row"], 403)
 
-        table_name = create_request.table
-        table_exists = await db.table_exists(table_name)
         columns = create_request.columns
         rows = create_request.rows_list
 
@@ -838,7 +872,7 @@ class TableCreateView(BaseView):
             # Must have insert-row permission
             if not await self.ds.allowed(
                 action="insert-row",
-                resource=DatabaseResource(database=database_name),
+                resource=table_resource,
                 actor=request.actor,
             ):
                 return Response.error(["Permission denied: need insert-row"], 403)
@@ -857,7 +891,7 @@ class TableCreateView(BaseView):
                 if create_request.alter:
                     if not await self.ds.allowed(
                         action="alter-table",
-                        resource=DatabaseResource(database=database_name),
+                        resource=table_resource,
                         actor=request.actor,
                     ):
                         return Response.error(
@@ -893,6 +927,7 @@ class TableCreateView(BaseView):
             )
 
         def create_table(conn):
+            check_structured_write_table(conn, table_name, allow_missing=True)
             db_for_write = sqlite_utils.Database(conn)
             table = db_for_write[table_name]
             if rows:
@@ -1012,6 +1047,9 @@ class DatabaseForeignKeyTargetsView(BaseView):
             for target in (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts()
             if target["fk_table"] not in hidden_tables
         ]
+        targets = await _filter_visible_foreign_key_targets(
+            self.ds, request.actor, database_name, targets
+        )
         return Response.json(
             {
                 "ok": True,
@@ -1050,6 +1088,15 @@ class TableForeignKeySuggestionsView(BaseView):
         source_columns, targets, current_by_column = await db.execute_fn(
             lambda conn: _foreign_key_suggestion_metadata(conn, table_name)
         )
+        targets = await _filter_visible_foreign_key_targets(
+            self.ds, request.actor, database_name, targets
+        )
+        visible_target_tables = {target["fk_table"] for target in targets}
+        current_by_column = {
+            column: current
+            for column, current in current_by_column.items()
+            if current["fk_table"] in visible_target_tables
+        }
 
         columns = []
         options_by_column = {}

datasette/views/table_extras.py

@@ -1206,7 +1206,10 @@ class ForeignKeyTablesExtra(Extra):
 
     async def resolve(self, context):
         return await context.foreign_key_tables(
-            context.database_name, context.table_name, context.pk_values
+            context.database_name,
+            context.table_name,
+            context.pk_values,
+            actor=context.request.actor,
         )
 
 

datasette/write_sql.py

@@ -83,6 +83,22 @@ def decision_for_write_sql_operation(
             )
     if operation.operation == "function":
         return IgnoreWriteSqlOperation("SQL function")
+    if (
+        operation.operation == "read"
+        and operation.target_type == "table"
+        and operation.table is not None
+        and operation.table_kind is None
+        and operation.table.lower().startswith("pragma_")
+    ):
+        # Eponymous table-valued PRAGMA functions (e.g. pragma_table_info("secret"))
+        # report a read of the synthetic "pragma_table_info" table, not of the
+        # table passed as an argument. That means a view-table denial on the real
+        # table is never consulted, so these could otherwise be used to read
+        # schema metadata (column names, table lists, ...) for tables the actor
+        # is not allowed to view. Reject them outright in untrusted write SQL,
+        # including inside CREATE VIEW bodies (whose reads are discovered here
+        # via the rolled-back dependency-read analysis above).
+        return UnsupportedWriteSqlOperation(unsupported_message)
     if (
         operation.operation == "read"
         and operation.target_type == "table"

demos/otel/Justfile

@@ -1,30 +0,0 @@
-# OpenTelemetry demo - see README.md in this directory.
-#
-#   just receiver  +  just serve     -> span summary in the terminal
-#   just jaeger    +  just serve     -> real trace UI at http://localhost:16686
-#
-# Both listen for OTLP/HTTP on port 4318, so `just serve` works with either
-# (but not both at once).
-
-otel_env := "OTEL_TRACES_EXPORTER=otlp OTEL_METRICS_EXPORTER=none OTEL_LOGS_EXPORTER=none OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 OTEL_SERVICE_NAME=datasette OTEL_BSP_SCHEDULE_DELAY=1000"
-
-default:
-    @just --list --unsorted
-
-# Terminal 1, option A: the ~150 line pure-Python receiver. Ctrl-C for a summary
-receiver:
-    uv run --with opentelemetry-proto python otlp_receiver.py
-
-# Terminal 1, option B: Jaeger from its own binary - no Docker. UI on :16686
-jaeger:
-    @command -v jaeger >/dev/null || { echo "No jaeger binary on PATH. Grab one from https://www.jaegertracing.io/download/"; exit 1; }
-    jaeger
-
-# Terminal 2: Datasette under the OpenTelemetry agent, exporting to :4318
-serve db="demo.db":
-    @[ "{{ db }}" != "demo.db" ] || [ -e demo.db ] || sqlite3 demo.db "create table plants(id integer primary key, name text, height_cm real); with recursive n(i) as (select 1 union all select i + 1 from n where i < 200) insert into plants select i, 'plant ' || i, abs(random() % 300) from n;"
-    {{ otel_env }} uv run --with opentelemetry-distro --with opentelemetry-exporter-otlp-proto-http opentelemetry-instrument datasette {{ db }} -p 8001
-
-# Terminal 3: make a traced request (defaults to the generated demo table)
-request path="/demo/plants":
-    curl -s -o /dev/null -w "%{http_code}\n" http://localhost:8001{{ path }}

demos/otel/README.md

@@ -1,121 +0,0 @@
-# OpenTelemetry demo
-
-Datasette core depends on `opentelemetry-api` only. It emits spans and nothing else — it never
-creates a `TracerProvider`, never configures an exporter, and never sets a sampler. With no SDK
-installed every span is a no-op and costs approximately nothing.
-
-That means "turning tracing on" is entirely the job of whoever runs Datasette. This directory
-shows two ways to do it, **neither of which needs Docker**:
-
-| | |
-|---|---|
-| `otlp_receiver.py` | A ~150 line pure-Python OTLP/HTTP receiver — a real protobuf export, summarized in your terminal |
-| `just jaeger` | The same export into Jaeger's own binary, for a real trace UI |
-
-Both listen for OTLP/HTTP on port 4318, so the Datasette side is identical — run one or the
-other, not both. The `Justfile` in this directory wraps every command below; bare `just` lists
-the recipes.
-
-## 1. A real OTLP export, pure Python
-
-Terminal 1 — the receiver (`just receiver`):
-
-```bash
-uv run --with opentelemetry-proto python demos/otel/otlp_receiver.py
-```
-
-Terminal 2 — Datasette under the OpenTelemetry agent (`just serve`, which also generates a
-200-row `demo.db` on first run):
-
-```bash
-OTEL_TRACES_EXPORTER=otlp \
-OTEL_METRICS_EXPORTER=none \
-OTEL_LOGS_EXPORTER=none \
-OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
-OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \
-OTEL_SERVICE_NAME=datasette \
-OTEL_BSP_SCHEDULE_DELAY=1000 \
-  uv run --with opentelemetry-distro \
-         --with opentelemetry-exporter-otlp-proto-http \
-    opentelemetry-instrument datasette demo.db -p 8001
-```
-
-Load a page (`just request`), wait a second for the batch flush, then Ctrl-C the receiver.
-Real output from startup plus one request against the 200-row demo table:
-
-```
-received 93 spans (93 total)
-
-=== 93 spans ===
-  db.query                                        43      18.55ms total
-  db.query.execute                                42       7.60ms total
-  db.write.queue_wait                              3       0.40ms total
-  db.write.execute                                 3       4.99ms total
-  datasette.startup                                1       7.17ms total
-  GET /(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)(\.(?P<format>\w+))?$    1      59.46ms total
-
-slowest spans:
-      59.46ms  GET /(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)(\.(?P<format>\w+))?$  -> 200
-       7.17ms  datasette.startup
-       4.58ms  db.write.execute
-       1.81ms  db.query  with limited as (select * from (select id, name, height_cm f
-
-2 root spans:
-  datasette.startup x1
-  GET /(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)(\.(?P<format>\w+))?$ x1
-```
-
-Things worth noticing:
-
-- **Exactly two root spans.** Every span belongs to either the request that caused it or to
-  `datasette.startup` — there are no orphans. A `block=False` write would be a third kind of
-  root, carrying a *link* back to the request that enqueued it rather than a parent, because
-  the write can outlive that request.
-- **Request spans are named after the matched route**, which in Datasette is a regex — that
-  string is ugly but it is the honest low-cardinality name. The pretty path is in the
-  `url.path` attribute.
-- **`db.query` vs `db.query.execute`.** The gap between the two is time spent waiting for a
-  thread. Likewise `db.write.queue_wait` vs `db.write.execute` is how you tell "the write was
-  slow" apart from "the write waited behind another writer".
-
-The receiver is a debugging aid, not a backend: nothing is persisted, it speaks OTLP/HTTP only
-(not gRPC), and it ignores metrics and logs.
-
-## 2. The same thing with a real UI: Jaeger, no Docker
-
-Jaeger ingests OTLP directly on the same port 4318, so the Datasette command does not change.
-With the `jaeger` binary on your PATH (<https://www.jaegertracing.io/download/>):
-
-```bash
-just jaeger    # terminal 1 — UI on http://localhost:16686
-just serve     # terminal 2 — identical to above
-just request   # terminal 3
-```
-
-Then open <http://localhost:16686>, pick service `datasette`, and Find Traces. Verified: one
-request produces exactly two traces — the request trace (~67 spans, rooted at the `GET ...`
-span, with every `db.query` nested inside it across thread boundaries) and the
-`datasette.startup` trace (~26 spans of catalog queries and connection warm-up).
-
-## Notes on the environment variables
-
-- **`opentelemetry-instrument` is required.** Setting `OTEL_TRACES_EXPORTER` and running plain
-  `datasette` produces nothing at all: that variable is read by the SDK's auto-configuration,
-  which only runs under the agent — core never installs a provider itself.
-- **`OTEL_SERVICE_NAME=datasette`** is what you pick from Jaeger's Service dropdown. Leave it
-  out and the SDK defaults to `unknown_service:<executable>`.
-- **`OTEL_BSP_SCHEDULE_DELAY=1000`** drops the batch flush from its ~10 second default to ~1
-  second. For a demo this is the difference between "it works" and "it looks broken". Do not
-  use it in production — it trades export efficiency for latency.
-- **`OTEL_METRICS_EXPORTER=none OTEL_LOGS_EXPORTER=none`** because `opentelemetry-distro`
-  defaults every signal to OTLP, and a traces-only backend like Jaeger answers the metrics
-  and logs exports with a stream of `StatusCode.UNIMPLEMENTED` noise.
-
-## Privacy
-
-`db.query.text` **is** recorded, truncated. SQL **parameter values are never recorded** — only
-a parameter count. On a public Datasette instance the SQL text is user-supplied; if you export
-to a third-party vendor, that text leaves your infrastructure.
-
-See the telemetry section of the Datasette documentation for the full span and attribute
-reference.

demos/otel/otlp_receiver.py

@@ -1,190 +0,0 @@
-"""
-A minimal OTLP/HTTP trace receiver, in about a hundred lines of Python.
-
-Run it, point Datasette's OpenTelemetry agent at it, and get a real end-to-end
-export - over the wire, in the real protobuf wire format - without Docker, a
-collector, or Jaeger:
-
-    # terminal 1
-    uv run --with opentelemetry-proto python demos/otel/otlp_receiver.py
-
-    # terminal 2
-    OTEL_TRACES_EXPORTER=otlp \\
-    OTEL_METRICS_EXPORTER=none \\
-    OTEL_LOGS_EXPORTER=none \\
-    OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \\
-    OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 \\
-    OTEL_SERVICE_NAME=datasette \\
-    OTEL_BSP_SCHEDULE_DELAY=1000 \\
-      uv run --with opentelemetry-distro \\
-             --with opentelemetry-exporter-otlp-proto-http \\
-        opentelemetry-instrument datasette mydb.db
-
-(or `just receiver` and `just serve mydb.db` from this directory.)
-
-Load a page, wait a second for the batch processor to flush, then press
-Ctrl-C here for a summary.
-
-This is a debugging aid, not a tracing backend: it does not persist anything,
-speaks only OTLP/HTTP (not gRPC), and ignores metrics and logs. For anything
-real, export to an actual backend instead.
-"""
-
-import gzip
-import signal
-import sys
-import threading
-from collections import Counter
-from http.server import BaseHTTPRequestHandler, HTTPServer
-
-try:
-    from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import (
-        ExportTraceServiceRequest,
-        ExportTraceServiceResponse,
-    )
-except ImportError:
-    sys.exit(
-        "This receiver needs the OpenTelemetry protobuf definitions:\n    uv run --with opentelemetry-proto python demos/otel/otlp_receiver.py"
-    )
-
-HOST = "127.0.0.1"
-PORT = 4318
-
-received = []
-
-
-def attribute_value(value):
-    for field in ("string_value", "int_value", "double_value", "bool_value"):
-        if value.HasField(field):
-            return getattr(value, field)
-    return None
-
-
-class OTLPHandler(BaseHTTPRequestHandler):
-    def log_message(self, *args):
-        pass  # the default handler logs every request to stderr
-
-    def do_GET(self):
-        # For the person who opens http://localhost:4318 in a browser
-        # expecting a UI: there isn't one here, on Jaeger either - 4318 is
-        # where exporters POST protobuf. Jaeger's UI lives on :16686.
-        body = (
-            f"This is an OTLP/HTTP ingestion endpoint ({len(received)} spans "
-            "received so far).\n\n"
-            "There is no UI on this port - OpenTelemetry exporters POST "
-            "protobuf to /v1/traces here.\nThe span summary appears in the "
-            "terminal running this receiver when you Ctrl-C it.\n"
-            "For a real UI, run Jaeger instead (`just jaeger`) and open "
-            "http://localhost:16686\n"
-        ).encode()
-        self.send_response(200)
-        self.send_header("Content-Type", "text/plain")
-        self.send_header("Content-Length", str(len(body)))
-        self.end_headers()
-        self.wfile.write(body)
-
-    def do_POST(self):
-        body = self.rfile.read(int(self.headers.get("Content-Length", 0)))
-        if self.headers.get("Content-Encoding") == "gzip":
-            body = gzip.decompress(body)
-
-        request = ExportTraceServiceRequest()
-        request.ParseFromString(body)
-        batch = 0
-        for resource_spans in request.resource_spans:
-            for scope_spans in resource_spans.scope_spans:
-                for span in scope_spans.spans:
-                    batch += 1
-                    received.append(
-                        {
-                            "name": span.name,
-                            "parent_id": span.parent_span_id.hex() or None,
-                            "duration_ms": (
-                                span.end_time_unix_nano - span.start_time_unix_nano
-                            )
-                            / 1e6,
-                            "attributes": {
-                                a.key: attribute_value(a.value) for a in span.attributes
-                            },
-                        }
-                    )
-        # flush=True so the live feedback survives being piped or redirected
-        print(f"received {batch} spans ({len(received)} total)", flush=True)
-
-        payload = ExportTraceServiceResponse().SerializeToString()
-        self.send_response(200)
-        self.send_header("Content-Type", "application/x-protobuf")
-        self.send_header("Content-Length", str(len(payload)))
-        self.end_headers()
-        self.wfile.write(payload)
-
-
-def span_detail(span):
-    "The one attribute most worth showing next to a span name."
-    attributes = span["attributes"]
-    query = attributes.get("db.query.text")
-    if query:
-        return f"  {query[:60]}"
-    status = attributes.get("http.response.status_code")
-    if status is not None:
-        return f"  -> {status}"
-    return ""
-
-
-def summarise():
-    if not received:
-        print("\nNo spans received.")
-        print("Remember: `opentelemetry-instrument` is required - Datasette core")
-        print("installs no provider - and the BatchSpanProcessor flushes about")
-        print("every 10s unless OTEL_BSP_SCHEDULE_DELAY says otherwise.")
-        return
-
-    print(f"\n=== {len(received)} spans ===")
-    for name, count in Counter(span["name"] for span in received).most_common():
-        total_ms = sum(s["duration_ms"] for s in received if s["name"] == name)
-        print(f"  {name:<45}{count:>5}  {total_ms:>9.2f}ms total")
-
-    slowest = sorted(received, key=lambda s: -s["duration_ms"])[:5]
-    print("\nslowest spans:")
-    for span in slowest:
-        print(f"  {span['duration_ms']:>9.2f}ms  {span['name']}{span_detail(span)}")
-
-    # Roots are one span per request (named "GET <route>"), datasette.startup,
-    # and any block=False write - those link back to their enqueuer rather than
-    # nesting under it, because the write can outlive the request that queued it.
-    roots = Counter(span["name"] for span in received if not span["parent_id"])
-    print(f"\n{sum(roots.values())} root spans:")
-    for name, count in roots.most_common():
-        print(f"  {name} x{count}")
-
-
-if __name__ == "__main__":
-    try:
-        server = HTTPServer((HOST, PORT), OTLPHandler)
-    except OSError as error:
-        sys.exit(
-            f"Could not listen on {HOST}:{PORT} ({error}).\n"
-            "Something else is already using the OTLP port - most likely an "
-            "earlier copy of this receiver, or Jaeger, still running."
-        )
-    print(f"OTLP/HTTP receiver listening on http://{HOST}:{PORT}")
-    print("Press Ctrl-C for a summary.")
-
-    # serve_forever() runs on a worker thread and the main thread just waits,
-    # so the summary still prints when this is launched through a wrapper such
-    # as `uv run`, where relying on KeyboardInterrupt alone is unreliable.
-    stop = threading.Event()
-    threading.Thread(target=server.serve_forever, daemon=True).start()
-
-    def request_stop(signum, frame):
-        stop.set()
-
-    signal.signal(signal.SIGINT, request_stop)
-    signal.signal(signal.SIGTERM, request_stop)
-
-    try:
-        stop.wait()
-    except KeyboardInterrupt:
-        pass
-    server.shutdown()
-    summarise()

docs/authentication.rst

@@ -158,6 +158,15 @@ Datasette resolves matching rules from most specific to least specific:
 
 This means a resource-level allow can provide an exception to a parent-level deny. It also means that two plugins which disagree at the same level resolve to deny.
 
+For table and view permissions, resource names use SQLite's case-insensitive
+identifier matching: ``Secret``, ``secret`` and ``SECRET`` identify the same
+table. This applies to configuration rules, plugin rules and token restrictions.
+Only ASCII letters are case-insensitive; non-ASCII characters remain distinct.
+Conflicting rules for different spellings of the same name follow the usual
+deny-wins rule at the same scope. Names retain their original spelling in
+resource listings and permission explanations. Database names, stored query
+names and other resource types remain case-sensitive.
+
 .. list-table:: Permission rule examples
    :header-rows: 1
 
@@ -182,6 +191,18 @@ This means a resource-level allow can provide an exception to a parent-level den
 
 The built-in public defaults are global allow rules for actions such as ``view-instance``, ``view-database`` and ``view-table``. They follow the same precedence rules as configuration and plugin rules. The ``--default-deny`` option prevents Datasette from contributing those default allow rules.
 
+The built-in ``datasette.default_permissions.sqlite_statistics`` plugin denies
+``view-table`` for ``sqlite_stat1``, ``sqlite_stat2``, ``sqlite_stat3`` and
+``sqlite_stat4``. These table-level denials also apply to root users and take
+precedence over configuration or plugin allow rules at the same scope.
+This controls table access and listings, without changing ``execute-sql`` or
+SQLite's internal use of statistics.
+
+A plugin can replace this policy by unregistering
+``datasette.default_permissions.sqlite_statistics`` through ``datasette.pm``
+and registering its own permission hook. Plugin registration is process-wide:
+replacing this policy affects every Datasette instance in that process.
+
 Datasette performs checks using :ref:`datasette_allowed`, which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``.
 
 ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified.
@@ -771,6 +792,8 @@ Datasette defaults to allowing any site visitor to execute their own custom SQL
 
 Access to this ability is controlled by the :ref:`actions_execute_sql` permission.
 
+This permission does not apply to structured table-browsing operations where Datasette constructs the SQL, such as sorting, column filters and :ref:`facets`. Faceting is controlled separately by the :ref:`setting_allow_facet` setting.
+
 The easiest way to disable arbitrary SQL queries is using the :ref:`default_allow_sql setting <setting_default_allow_sql>` when you first start Datasette running.
 
 You can alternatively use an ``"allow_sql"`` block to control who is allowed to execute arbitrary SQL queries.
@@ -1359,6 +1382,12 @@ view-table
 
 Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.io/fixtures/complex_foreign_keys
 
+Derived implementation tables require access to their immediate source: FTS and RTree shadow tables require access to their virtual table, external-content FTS tables require access to their content table, and FTS vocabulary tables (``fts5vocab`` and ``fts4aux``) require access to their FTS table. The derived table's own permission rules also apply.
+
+Access is always denied if the source table is itself derived, or if a vocabulary table's source cannot be identified.
+
+The same rules apply to individual permission checks and table listings, including whether they are private. If a database error prevents dependency discovery, the check or listing fails with an error instead of ignoring the dependencies. Failed discovery results are not cached, so later checks can retry.
+
 ``resource`` - ``datasette.resources.TableResource(database, table)``
     ``database`` is the name of the database (string)
 
@@ -1521,6 +1550,8 @@ execute-sql
 
 Actor is allowed to run arbitrary read-only SQL queries against a specific database using the :ref:`custom SQL query page <pages_custom_sql_queries>`, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100
 
+This action also controls raw SQL supplied using ``?_where=``. It does not control structured table-browsing features such as :ref:`facets`, which use SQL generated by Datasette and are controlled by :ref:`setting_allow_facet`.
+
 ``resource`` - ``datasette.resources.DatabaseResource(database)``
     ``database`` is the name of the database (string)
 

docs/changelog.rst

@@ -4,14 +4,54 @@
 Changelog
 =========
 
-.. _v_unreleased:
+.. _unreleased:
 
 Unreleased
 ----------
 
-- Datasette's database layer now emits `OpenTelemetry <https://opentelemetry.io/>`__ spans: one per query, covering the full round trip including time spent waiting for a SQL worker thread, plus separate child spans for the execution itself and for time spent in the write queue. Datasette core depends on ``opentelemetry-api`` only and never installs an SDK provider, an exporter or a sampler, so there is no effect and no measurable overhead unless tracing is switched on externally - normally with the standard ``opentelemetry-instrument`` agent. See :ref:`internals_telemetry`. (:issue:`1730`)
-- :ref:`db.execute(sql, ..., table=None) <database_execute>` has a new optional ``table=`` parameter, naming the table a query is about so it can be recorded on that query's OpenTelemetry span. It has no effect on query execution, and Datasette never derives it from the SQL. (:issue:`1730`)
-- **Breaking change:** Datasette's hand-rolled tracer has been removed, now that OpenTelemetry covers the same ground. The ``?_trace=1`` query string parameter, the ``trace_debug`` setting and the ``datasette.tracer`` module are all gone. ``datasette.tracer.trace()`` and ``datasette.tracer.trace_child_tasks()`` were documented plugin APIs, so any plugin importing them will now raise ``ModuleNotFoundError`` and needs a new release. `datasette-pretty-traces <https://datasette.io/plugins/datasette-pretty-traces>`__ does not import that module, but it renders ``?_trace=1`` output, so it no longer has anything to display. (:issue:`1730`)
+- Datasette now uses `httpx2 <https://httpx2.pydantic.dev/>`__, the Pydantic-maintained continuation of `httpx <https://www.python-httpx.org/>`__, in place of ``httpx``. The public API is the same, but responses returned by :ref:`internals_datasette_client` are now ``httpx2.Response`` objects rather than ``httpx.Response``. Plugins that use ``isinstance()`` checks against ``httpx.Response`` should be updated to use ``httpx2``. **Plugins that use httpx without explicitly depending on it** will need to add an explicit dependency or switch to `httpx2`.
+
+.. _v1_0_a39:
+
+1.0a39 (2026-09-10)
+-------------------
+
+This alpha release includes security fixes for permissions, SQL construction, HTML rendering, authentication and caching, plus improvements to application startup and write execution.
+
+See `0.65.4 <https://docs.datasette.io/en/stable/changelog.html#v0-65-4>`__ for fixes that have been backported to the stable 0.65.x branch.
+
+The Datasette blog `has more details on these releases <https://datasette.io/blog/2026/september-security-releases/>`__.
+
+Some of the security fixes include:
+
+- Table and view permission checks now take SQLite's case-insensitive names into account. See :ref:`authentication_permissions_explained`.
+- Viewing a full-text search index table now checks you have permission to view the table from which it draws its content.
+- Viewing SQLite statistics tables (``sqlite_stat1`` through ``sqlite_stat4``) is now denied by a default.
+- Table schema display now obeys the ``view-table`` permission.
+- Table filters using ``?_through=`` require permission to view the intermediate table.
+- Foreign-key target and suggestion APIs, incoming foreign-key relationships and their row counts now respect ``view-table`` permission.
+- Row endpoints check permissions before resolving primary keys, to avoid revealing the existence of an otherwise invisible primary key.
+- Improved permission checks for the create-table API. See :ref:`json_api_write`.
+- The write SQL interface now checks ``view-table`` permission for tables referenced by ``CREATE VIEW`` statements.
+- Fixed SQL identifier escaping for column names from untrusted database schemas.
+- Fixed HTML escaping for column names from untrusted database schemas.
+- URL columns now render links only for validated HTTP or HTTPS URLs.
+- Private and personalized dynamic responses now use ``Cache-Control: private, no-store``. Anonymous dynamic responses vary by ``Cookie`` and ``Authorization``.
+- Actor cookies now respect ``expire_after``.
+- Restricted actors can no longer create API tokens.
+- Stored-query create, edit and delete forms now block framing to prevent clickjacking.
+- Configuration secret redaction now matches key names case-insensitively.
+- SQLite extension loading is disabled after extensions supplied using ``--load-extension`` have been loaded.
+
+Other improvements and fixes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- :ref:`db.execute_write() <database_execute_write>` now has a default execution time limit of 2,000ms. Plugins can override this using ``time_limit_ms=`` or disable it using ``time_limit_ms=None``. This limit is independent of the ``sql_time_limit_ms`` setting for read queries.
+- Application startup now runs through ASGI lifespan events before requests are accepted, with a first-request fallback for hosts without lifespan support. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:pr:`2887`)
+- ``datasette serve`` now runs startup hooks and Uvicorn on the same event loop, preserving background tasks started by plugins. The minimum Uvicorn version is now 0.29. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:pr:`2886`)
+- Non-blocking writes using ``execute_write_fn(..., block=False)`` now return a distinct task UUID for every call and work correctly with ``num_sql_threads=0``. Thanks, `Zain Dana Harper <https://github.com/HarperZ9>`__. (:issue:`2860`, :issue:`2859`)
+- Dropping a table now disables its full-text search index first. (:issue:`2874`)
+- Fixed ``CREATE VIEW`` SQL analysis on Python 3.10.
 
 .. _v1_0_a38:
 
@@ -1160,7 +1200,7 @@ Datasette also now requires Python 3.7 or higher.
 - ``sqlite_stat`` tables are now hidden by default. (:issue:`1587`)
 - SpatiaLite tables ``data_licenses``, ``KNN`` and ``KNN2`` are now hidden by default. (:issue:`1601`)
 - SQL query tracing mechanism now works for queries executed in ``asyncio`` sub-tasks, such as those created by ``asyncio.gather()``. (:issue:`1576`)
-- ``datasette.tracer`` mechanism is now documented.
+- :ref:`internals_tracer` mechanism is now documented.
 - Common Datasette symbols can now be imported directly from the top-level ``datasette`` package, see :ref:`internals_shortcuts`. Those symbols are ``Response``, ``Forbidden``, ``NotFound``, ``hookimpl``, ``actor_matches_allow``. (:issue:`957`)
 - ``/-/versions`` page now returns additional details for libraries used by SpatiaLite. (:issue:`1607`)
 - Documentation now links to the `Datasette Tutorials <https://datasette.io/tutorials>`__.
@@ -1326,7 +1366,7 @@ New features
 - ``?_facet_size=max`` sets that to the maximum, which defaults to 1,000 and is controlled by the the :ref:`setting_max_returned_rows` setting. If facet results are truncated the … at the bottom of the facet list now links to this parameter. (:issue:`1337`)
 - ``?_nofacet=1`` option to disable all facet calculations on a page, used as a performance optimization for CSV exports and ``?_shape=array/object``. (:issue:`1349`, :issue:`263`)
 - ``?_nocount=1`` option to disable full query result counts. (:issue:`1353`)
-- ``?_trace=1`` debugging option is now controlled by the new ``trace_debug`` setting, which is turned off by default. (:issue:`1359`)
+- ``?_trace=1`` debugging option is now controlled by the new :ref:`setting_trace_debug` setting, which is turned off by default. (:issue:`1359`)
 
 Bug fixes and other improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/cli-reference.rst

@@ -283,6 +283,8 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam
                                    protocol (default=False)
       template_debug               Allow display of template debug information with
                                    ?_context=1 (default=False)
+      trace_debug                  Allow display of SQL trace debug information with
+                                   ?_trace=1 (default=False)
       base_url                     Datasette URLs should use this base path
                                    (default=/)
 

docs/facets.rst

@@ -14,6 +14,8 @@ Here's `an example <https://congress-legislators.datasettes.com/legislators/legi
 
 Facets can be specified in two ways: using query string parameters, or in ``metadata.json`` configuration for the table.
 
+Facet queries are generated by Datasette and summarize rows the actor already has permission to view. They do not require the :ref:`actions_execute_sql` permission. Use the :ref:`setting_allow_facet` setting to control whether users can request facets using query string parameters.
+
 Facets in query strings
 -----------------------
 

docs/full_text_search.rst

@@ -52,11 +52,11 @@ Configuring full-text search for a table or view
 
 If a table has a corresponding FTS table set up using the ``content=`` argument to ``CREATE VIRTUAL TABLE`` shown below, Datasette will detect it automatically and add a search interface to the table page for that table.
 
-You can also manually configure which table should be used for full-text search using query string parameters or table configuration in ``datasette.yaml`` (see :ref:`table_configuration_fts`). You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option.
+You can also manually configure which table should be used for full-text search using table configuration in ``datasette.yaml`` (see :ref:`table_configuration_fts`). You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option.
 
-Use ``?_fts_table=x`` to over-ride the FTS table for a specific page. If the primary key was something other than ``rowid`` you can use ``?_fts_pk=col`` to set that as well. This is particularly useful for views, for example:
+The legacy ``?_fts_table=x`` and ``?_fts_pk=col`` query string parameters are accepted only if they exactly match the configured or automatically detected FTS mapping. They cannot be used to select a different FTS table or primary key. This prevents a public table from being used to probe the contents of a private FTS table.
 
-https://latest.datasette.io/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk
+Searching also requires the current actor to have ``view-table`` permission for the FTS table itself, in addition to permission to view the table or view being searched.
 
 The ``fts_table`` metadata property can be used to specify an associated FTS table. If the primary key column in your table which was used to populate the FTS table is something other than ``rowid``, you can specify the column to use with the ``fts_pk`` property.
 

docs/internals.rst

@@ -1594,32 +1594,32 @@ datasette.client
 
 Plugins can make internal simulated HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins, while avoiding the overhead of making an external HTTP call to access those APIs.
 
-The ``datasette.client`` object is a wrapper around the `HTTPX Python library <https://www.python-httpx.org/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
+The ``datasette.client`` object is a wrapper around the `HTTPX2 Python library <https://httpx2.pydantic.dev/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
 
 It offers the following methods:
 
-``await datasette.client.get(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.get(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal GET request against that path.
 
-``await datasette.client.post(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.post(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal POST request. Use ``data={"name": "value"}`` to pass form parameters.
 
-``await datasette.client.options(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.options(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal OPTIONS request.
 
-``await datasette.client.head(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.head(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal HEAD request.
 
-``await datasette.client.put(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.put(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal PUT request.
 
-``await datasette.client.patch(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.patch(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal PATCH request.
 
-``await datasette.client.delete(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.delete(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal DELETE request.
 
-``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal request with the given HTTP method against that path.
 
 These methods can be used with :ref:`internals_datasette_urls` - for example:
@@ -1636,7 +1636,7 @@ These methods can be used with :ref:`internals_datasette_urls` - for example:
 
 ``datasette.client`` methods automatically take the current :ref:`setting_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path.
 
-For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation <https://www.python-httpx.org/async/>`__.
+For documentation on available ``**kwargs`` options and the shape of the HTTPX2 Response object refer to the `HTTPX2 Async documentation <https://httpx2.pydantic.dev/async/>`__.
 
 .. _internals_datasette_client_actor:
 
@@ -1963,9 +1963,6 @@ Executes a SQL query against the database and returns the resulting rows (see :r
 ``log_sql_errors`` - boolean
     Should any SQL errors be logged to the console in addition to being raised as an error? Defaults to ``True``.
 
-``table`` - string
-    The name of the table this query is about, if the caller already knows it. This has no effect on how the query executes - it is recorded as the ``db.collection.name`` attribute on the :ref:`OpenTelemetry span <internals_telemetry>` for the query. Datasette never derives this from the SQL, so leave it unset for queries that do not have one obvious table.
-
 .. _database_results:
 
 Results
@@ -2026,8 +2023,8 @@ Example usage:
 
 .. _database_execute_write:
 
-await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True)
---------------------------------------------------------------------------------------------------------------------------
+await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True, time_limit_ms=2000)
+----------------------------------------------------------------------------------------------------------------------------------------------
 
 SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received.
 
@@ -2066,6 +2063,13 @@ Each call to ``execute_write()`` will be executed inside a transaction. Pass
 ``transaction=False`` for statements such as ``VACUUM`` that cannot run inside
 a transaction.
 
+Write statements have a default time limit of 2,000ms. Pass a different value
+using ``time_limit_ms=`` or use ``time_limit_ms=None`` to allow the statement to
+run without a time limit.
+
+This write limit is independent of the ``sql_time_limit_ms`` setting used for
+read queries. Changing that setting does not change the default write limit.
+
 .. _database_execute_write_script:
 
 await db.execute_write_script(sql, block=True)
@@ -2316,178 +2320,6 @@ The ``Database`` class also provides properties and methods for introspecting th
           }
         }
 
-.. _internals_telemetry:
-
-OpenTelemetry
-=============
-
-Datasette core depends on `opentelemetry-api <https://pypi.org/project/opentelemetry-api/>`__ only. It never creates a ``TracerProvider``, never configures an exporter and never sets a sampler. With no OpenTelemetry SDK provider installed, every span described below is a no-op ``NonRecordingSpan``: nothing is recorded, nothing is exported, and the cost does not show up in page latency. Benchmarking a table page with and without this instrumentation, the median moved by less than the run-to-run variation of the benchmark itself.
-
-Turning tracing on is entirely an operational decision made outside of Datasette itself: run Datasette under the standard ``opentelemetry-instrument`` agent, or embed Datasette inside a host application that installs its own provider.
-
-Everything Datasette emits carries the instrumentation scope ``datasette``, versioned with the running Datasette version and declaring the `semantic conventions schema <https://opentelemetry.io/docs/specs/otel/schemas/>`__ its attribute names follow.
-
-.. _internals_telemetry_turning_on:
-
-Turning tracing on
-------------------
-
-Install an OpenTelemetry SDK, an exporter and the instrumentation agent, then launch Datasette through ``opentelemetry-instrument``:
-
-.. code-block:: bash
-
-    pip install opentelemetry-distro opentelemetry-exporter-otlp
-    OTEL_SERVICE_NAME=datasette \
-    OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \
-    OTEL_METRICS_EXPORTER=none \
-    OTEL_LOGS_EXPORTER=none \
-      opentelemetry-instrument datasette mydb.db
-
-Point ``OTEL_EXPORTER_OTLP_ENDPOINT`` at whichever tracing backend you use. To print spans straight to the terminal instead, with no backend at all, drop that variable and set ``OTEL_TRACES_EXPORTER=console`` in its place.
-
-A few things catch people out the first time:
-
-.. warning::
-    ``OTEL_TRACES_EXPORTER=console datasette mydb.db`` produces **nothing**. That environment variable is read by the OpenTelemetry SDK's auto-configuration, which only runs when the ``opentelemetry-instrument`` agent wraps the process. Datasette core installs no provider, so a plain ``datasette`` process emits nothing at all, whatever ``OTEL_`` variables are set.
-
-Spans do not appear immediately. The SDK's default ``BatchSpanProcessor`` flushes on a timer, every 5 seconds. Either wait, or stop the process - shutdown triggers a final flush - or set ``OTEL_BSP_SCHEDULE_DELAY=1000`` while you are experimenting. That last one is for demos, not for production.
-
-Always set ``OTEL_SERVICE_NAME``. Without it the SDK's default resource reports a ``service.name`` of ``unknown_service``, and your traces will be filed under that instead of under a name you can search for.
-
-Setting ``OTEL_METRICS_EXPORTER=none`` and ``OTEL_LOGS_EXPORTER=none`` is worth doing unless your backend accepts those signals too - ``opentelemetry-distro`` defaults every signal to OTLP, and a traces-only backend will reject the other two noisily. Datasette itself emits no metrics and no logs through OpenTelemetry.
-
-Span reference
---------------
-
-Datasette emits six spans. One covers the HTTP request, and is the root everything else raised while serving that request hangs from. Four describe the database layer - one per query, one for the work that query does inside a SQL worker thread, and two more for the write queue. The sixth covers startup. Attribute names use the ``datasette.*`` prefix for Datasette-specific data, alongside standard OpenTelemetry attributes such as ``db.system``.
-
-This reference is generated from ``datasette/telemetry_registry.py``, the single source of truth for every span and attribute Datasette emits. A conformance test makes real requests and compares what is actually emitted against that registry in both directions, so nothing here is hand-maintained and nothing can silently drift out of date.
-
-Spans are ``SpanKind.INTERNAL`` unless a kind is listed below. Two are not: the request span is ``SERVER``, and ``db.query`` is ``CLIENT`` because it is the one span that represents a call to a database rather than Datasette's own work. Trace UIs use the kind to decide whether to render a span as an inbound request or as a database call. ``db.query``'s children stay ``INTERNAL`` because they are Datasette's decomposition of that one query - marking them ``CLIENT`` too would make a single query look like several database calls to anything counting by kind.
-
-The request span's name is the only one that is not a fixed string - it is composed from the request, so the heading below shows the template rather than a literal you will see in a trace. A request to a table page produces a span named, in full::
-
-    GET /(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)(\.(?P<format>\w+))?$
-
-That is the route's compiled regular expression, not a prettified ``/{database}/{table}`` template. It is deliberate: Datasette routes with compiled patterns and the route table is fixed when the app is built, so the pattern is exact, bounded and needs no parsing, while transforming it into something prettier accretes edge cases. Django's own instrumentation ships regex-flavoured routes for the same reason.
-
-.. [[[cog
-    from telemetry_doc import spans
-    spans(cog)
-.. ]]]
-
-``{http.request.method} {http.route}``
-    One span per HTTP request, created by the outermost layer of the ASGI stack - so plugin ``asgi_wrapper()`` middleware, CSRF protection and every database span raised while serving the request all nest inside it. Without it each of those would be its own root trace. The span name is not a fixed string: it is the method followed by the matched route, and just the method for a request that matched no route. The span starts at the ASGI edge, before routing has happened, so it is named for the method there and renamed once the route is known. W3C ``traceparent`` and ``baggage`` headers are extracted using the global propagator, so a request arriving from an already-traced caller continues that trace; set ``OTEL_PROPAGATORS=none`` to turn that off, and strip those headers at your proxy if your instance is public.
-
-    Kind: ``SERVER``.
-
-    Attributes:
-
-    - ``http.request.method`` - The HTTP method, clamped to the nine methods RFC 9110 and RFC 5789 define. Anything else is reported as ``_OTHER``: the method is a client-controlled string, so echoing it back unbounded would be a cardinality hazard.
-    - ``http.route`` *(optional)* - The route the request matched, as the compiled regular expression pattern Datasette routes with - for example ``/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)(\.(?P<format>\w+))?$`` for a table page. It is deliberately the pattern rather than a prettified ``/{database}/{table}`` template: the route table is fixed when the app is built, so the pattern is exact, bounded and needs no parsing, whereas the transform into something prettier accretes edge cases. Unlike ``url.path`` this is low cardinality, so it is the attribute to group by. Omitted when no route matched - a 404 - which is also when the span name falls back to the bare method.
-    - ``url.path`` - The path portion of the URL. The query string is deliberately **not** recorded, on this or any other span: Datasette puts user-supplied SQL in ``?sql=`` and canned query parameters in the query string, so exporting it by default would export exactly the data the rest of this instrumentation is careful with.
-    - ``url.scheme`` - ``http`` or ``https``.
-    - ``server.address`` *(optional)* - The ``Host`` header. Client-controlled, so treat it as untrusted input rather than as the identity of the server.
-    - ``user_agent.original`` *(optional)* - The ``User-Agent`` header, verbatim. Omitted if the client sent none. The client's IP address is deliberately not recorded: core records no identifier that would tie a span to a person.
-    - ``http.response.status_code`` *(optional)* - The status of the response, read from the ASGI ``http.response.start`` message rather than from a :ref:`internals_response` object - several views, including static files, file downloads and streaming CSV, send that message themselves and never build one. Omitted if the connection closed before anything was sent.
-    - ``error.type`` *(optional)* - Set when the request failed: the exception class name if one escaped the application, otherwise the status code as a string for a 5xx response. A 4xx does **not** set this and does not set an error status - per semantic conventions a client error is not a server span's failure.
-
-``db.query``
-    A SQL operation issued by Datasette, covering the full round trip including any time spent queued for a thread.
-
-    Kind: ``CLIENT``.
-
-    Attributes:
-
-    - ``db.system`` - Always ``sqlite``.
-    - ``db.namespace`` - Name of the database being queried.
-    - ``db.query.text`` - The SQL, truncated to 2048 characters. Never the parameter values.
-    - ``db.operation.name`` *(optional)* - The statement's leading keyword - ``SELECT``, ``INSERT``, ``CREATE``, and so on - matched against a small fixed allowlist. Omitted rather than set to an arbitrary value: the allowlist exists because this attribute is a candidate dimension for a query-duration metric in a later phase, and echoing an unrecognised first token from user-supplied SQL would be an unbounded-cardinality hazard. Also omitted for ``execute_write_script()``, which runs multiple statements - per semantic conventions, the operation name should not be extracted from query text that can contain more than one operation. Note that a statement beginning with a CTE reports ``WITH``, not the operation inside it - a substantial share of Datasette's own reads take that form. Resolving it further would mean parsing.
-    - ``db.collection.name`` *(optional)* - The primary table, set only where the view already knows it - the table and row pages. Omitted for arbitrary ``?sql=`` queries, where determining the table would mean parsing the query.
-    - ``datasette.param_count`` *(optional)* - Number of bound parameters. Recorded instead of the values themselves.
-    - ``datasette.param_sets`` *(optional)* - Number of parameter sets consumed by ``execute_write_many()``. Not a row count - ``executemany()`` returns no rows. The parameter values themselves are never recorded: that sequence can hold thousands of rows.
-    - ``datasette.time_limit_ms`` *(optional)* - The :ref:`setting_sql_time_limit_ms` value this query ran under. Set on reads, which are the queries that time limit applies to.
-    - ``datasette.rows_returned`` *(optional)* - Number of rows a read returned. Set on the read path only, and only when the read succeeded.
-    - ``datasette.truncated`` *(optional)* - True if the result was cut short by :ref:`setting_max_returned_rows`.
-    - ``datasette.interrupted`` *(optional)* - True if the query was cancelled for exceeding the time limit. The span status is also set to ``ERROR``, unless the caller asked for a budget shorter than :ref:`setting_sql_time_limit_ms` - as table counts, facet suggestion and autocomplete all do - in which case running out of time is an expected answer rather than a failure and the status is left unset.
-    - ``datasette.sql_error_suppressed`` *(optional)* - True when the query failed but the caller passed ``log_sql_errors=False``, meaning it was probing and treats failure as an expected answer. Facet suggestion does this against every column.
-    - ``datasette.executescript`` *(optional)* - True for ``execute_write_script()``, which runs multiple statements.
-    - ``datasette.executemany`` *(optional)* - True for ``execute_write_many()``, which runs one statement against many parameter sets.
-
-``db.query.execute``
-    The read executing inside a SQL worker thread. Child of ``db.query``; the gap between the two is time spent waiting for a thread.
-
-    No attributes.
-
-``db.write.queue_wait``
-    Time a write spent waiting in its database's write queue before the write thread picked it up. Child of ``db.query`` for a ``block=True`` write, where the caller awaits the write and containment is accurate. For a ``block=False`` write the caller does not await it - the enqueueing request *caused* the write without *containing* it, and the write's spans can outlive the request's own - so this is a root span instead, carrying an OpenTelemetry link back to the enqueueing span rather than a parent. A link records causation without asserting containment, which is exactly the distinction here.
-
-    No attributes.
-
-``db.write.execute``
-    The write executing on the write thread. Child of ``db.query`` for a ``block=True`` write; for ``block=False`` a root span with a link back to the enqueueing span instead - see ``db.write.queue_wait`` above.
-
-    Attributes:
-
-    - ``datasette.isolated_connection`` - True if the write ran on its own connection rather than the shared write connection.
-    - ``datasette.transaction`` - False for statements such as ``VACUUM`` that cannot run inside a transaction.
-
-``datasette.startup``
-    ``invoke_startup()`` running: ``register_events``, ``register_actions``, ``register_column_types``, ``prepare_jinja2_environment``, internal-database schema catalog refresh (including the ``prepare_connection`` warm-up this triggers for each database touched for the first time), saved queries, column type config and the ``startup`` hook. Runs once per process, before any request exists, so without this span every child it creates would be its own orphan root trace. A connection warmed later - lazily, the first time a *request* touches a new database or thread - nests under that request's own span instead, not under this one, since this span has already ended by then.
-
-    No attributes.
-
-.. [[[end]]]
-
-.. _internals_telemetry_requests:
-
-Requests and inbound trace context
-----------------------------------
-
-Datasette creates the request span itself, at the outermost layer of the ASGI stack, so a trace is complete out of the box with no plugin and no extra instrumentation package. Everything raised while serving the request - plugin ``asgi_wrapper()`` middleware, CSRF protection, every database query - nests inside it.
-
-**Inbound trace context is trusted by default.** W3C ``traceparent`` and ``baggage`` headers are extracted from every request using the global propagator, so a request arriving from an already-traced caller continues that trace instead of starting a new one. That is what every other framework instrumentation does - Flask, Django, FastAPI and ``opentelemetry-instrumentation-asgi`` all extract unconditionally - but on an instance open to the internet it means an arbitrary client can influence your traces:
-
-- **Trace-ID pollution.** The client chooses the trace ID its request is filed under.
-- **Sampling control.** The SDK's default sampler is ``parentbased_always_on``, so under any parent-based sampler a client's sampled flag can force recording - a telemetry-cost denial of service - or suppress it.
-- **Baggage injection**, through the default composite propagator.
-
-Because extraction goes through the *global* propagator there is no Datasette setting to configure, and the remedies are the standard OpenTelemetry ones:
-
-- Strip ``traceparent``, ``tracestate`` and ``baggage`` at your reverse proxy, which is the right answer for a public instance fronted by one.
-- Set ``OTEL_PROPAGATORS=none`` to disable extraction entirely, or ``OTEL_PROPAGATORS=tracecontext`` to keep trace continuation and drop baggage.
-- Use a sampler that is not parent-based, which neutralises the sampling concern on its own.
-
-**Installing an ASGI instrumentation as well is harmless.** If you wire up ``opentelemetry-instrumentation-asgi`` through an ``asgi_wrapper()`` plugin, its middleware lands *inside* Datasette's own, so its span becomes a redundant child ``SERVER`` span in the same trace. Nothing is re-orphaned. There is no setting to turn Datasette's request span off, because "turn it off" is already covered by installing no provider, or by ``OTEL_SDK_DISABLED=true``.
-
-**Where** ``datasette.startup`` **lands depends on how you run Datasette.** ``datasette serve`` calls ``invoke_startup()`` before the server starts accepting connections, so the startup span is its own trace. An ASGI-hosted or programmatic deployment reaches startup lazily, on the first request, so there the startup span nests under that first request - which is honest, since it genuinely is that request's latency.
-
-.. _internals_telemetry_privacy:
-
-Privacy and safety
-------------------
-
-Spans leave your infrastructure whenever you configure an exporter, so what goes into them is a security decision. Datasette's rules are:
-
-- **SQL text is truncated to 2048 characters.** On a public instance the SQL is supplied by visitors and is unbounded in length, so ``db.query.text`` is cut off - with a ``…[truncated]`` marker - rather than allowed to set the size of a span.
-- **SQL parameter values are never recorded.** Only ``datasette.param_count``, a count. Parameter values are the part of a query most likely to hold something sensitive, and separating them from the SQL is the reason bound parameters exist.
-- **No actor identifiers are recorded.** No actor ID, no actor JSON, no client IP address. Nothing on a span identifies who made the request.
-- **Table names come only from an explicit** ``table=`` **argument.** ``db.collection.name`` is set by callers that already know which table they are working with, and is never derived from the SQL. Deriving it would mean parsing, and on an instance where visitors can create tables the set of possible values has no ceiling.
-- **The query string is never recorded.** There is no ``url.query`` attribute on the request span or on any other span. Datasette puts user-supplied SQL in ``?sql=`` and canned query parameters in the query string, so recording it by default would export exactly the class of data the rules above are careful with. Only ``url.path`` and ``http.route`` are recorded.
-
-The SQL itself, though, *is* recorded, and on a public instance that means anything a visitor types into the query editor or passes as ``?sql=`` will be exported along with the span. That is the trade-off tracing a query engine makes.
-
-.. _internals_telemetry_limitations:
-
-Known limitations
------------------
-
-- ``http.route`` **is a compiled regular expression, not a pretty route template.** See :ref:`internals_telemetry_requests` above for why.
-- **Inbound trace context is trusted by default**, which on a public instance means a client can influence your trace IDs, your sampling and your baggage. :ref:`internals_telemetry_requests` lists the remedies.
-- **Two plugin hooks run outside the** ``datasette.startup`` **span.** ``register_output_renderer`` is dispatched from ``Datasette.__init__()`` and ``asgi_wrapper`` from ``Datasette.app()``, both of which happen before ``invoke_startup()``. Datasette itself queries no database in either, so a default install emits nothing there - but a plugin that does will produce a root trace. Covering these would mean holding a span open across object construction, which is worse than the orphan.
-- ``db.operation.name`` **reports** ``WITH`` **for a statement that opens with a common table expression**, rather than the operation inside it, and a substantial share of Datasette's own reads take that form. The attribute is a leading-keyword match against a fixed allowlist, deliberately not a parse.
-- **Spans emitted before a provider is installed are not recorded.** If you are embedding Datasette in a host application, install your ``TracerProvider`` before serving traffic. This is ordinary OpenTelemetry behaviour rather than anything Datasette controls; nothing is permanently affected, those particular spans are simply dropped.
-
 .. _internals_csrf:
 
 CSRF protection
@@ -2771,6 +2603,8 @@ Async version of :ref:`call_with_supported_arguments <internals_utils_call_with_
 
 .. autofunction:: datasette.utils.async_call_with_supported_arguments
 
+.. _internals_tracer:
+
 JSON encoding
 -------------
 
@@ -2778,6 +2612,82 @@ JSON encoding
 
 .. autoclass:: datasette.utils.CustomJSONEncoder
 
+datasette.tracer
+================
+
+Running Datasette with ``--setting trace_debug 1`` enables trace debug output, which can then be viewed by adding ``?_trace=1`` to the query string for any page.
+
+You can see an example of this at the bottom of `latest.datasette.io/fixtures/facetable?_trace=1 <https://latest.datasette.io/fixtures/facetable?_trace=1>`__. The JSON output shows full details of every SQL query that was executed to generate the page.
+
+The `datasette-pretty-traces <https://datasette.io/plugins/datasette-pretty-traces>`__ plugin can be installed to provide a more readable display of this information. You can see `a demo of that here <https://latest-with-plugins.datasette.io/github/commits?_trace=1>`__.
+
+You can add your own custom traces to the JSON output using the ``trace()`` context manager. This takes a string that identifies the type of trace being recorded, and records any keyword arguments as additional JSON keys on the resulting trace object.
+
+The start and end time, duration and a traceback of where the trace was executed will be automatically attached to the JSON object.
+
+This example uses trace to record the start, end and duration of any HTTP GET requests made using the function:
+
+.. code-block:: python
+
+    from datasette.tracer import trace
+    import httpx2
+
+
+    async def fetch_url(url):
+        with trace("fetch-url", url=url):
+            async with httpx2.AsyncClient() as client:
+                return await client.get(url)
+
+.. _internals_tracer_trace_child_tasks:
+
+Tracing child tasks
+-------------------
+
+If your code uses a mechanism such as ``asyncio.gather()`` to execute code in additional tasks you may find that some of the traces are missing from the display.
+
+You can use the ``trace_child_tasks()`` context manager to ensure these child tasks are correctly handled.
+
+.. code-block:: python
+
+    from datasette import tracer
+
+    with tracer.trace_child_tasks():
+        results = await asyncio.gather(
+            # ... async tasks here
+        )
+
+This example uses the :ref:`register_routes() <plugin_register_routes>` plugin hook to add a page at ``/parallel-queries`` which executes two SQL queries in parallel using ``asyncio.gather()`` and returns their results.
+
+.. code-block:: python
+
+    from datasette import hookimpl
+    from datasette import tracer
+
+
+    @hookimpl
+    def register_routes():
+        async def parallel_queries(datasette):
+            db = datasette.get_database()
+            with tracer.trace_child_tasks():
+                one, two = await asyncio.gather(
+                    db.execute("select 1"),
+                    db.execute("select 2"),
+                )
+            return Response.json(
+                {
+                    "one": one.single_value(),
+                    "two": two.single_value(),
+                }
+            )
+
+        return [
+            (r"/parallel-queries$", parallel_queries),
+        ]
+
+Note that running parallel SQL queries in this way has `been known to cause problems in the past <https://github.com/simonw/datasette/issues/2189>`__, so treat this example with caution.
+
+Adding ``?_trace=1`` will show that the trace covers both of those child tasks.
+
 .. _internals_shortcuts:
 
 Import shortcuts

docs/introspection.rst

@@ -124,6 +124,7 @@ Shows the :ref:`configuration <configuration>` for this instance of Datasette. T
         "ok": true,
         "settings": {
             "template_debug": true,
+            "trace_debug": true,
             "force_https_urls": true
         }
     }

docs/json_api.rst

@@ -313,6 +313,16 @@ query string arguments:
     For how many seconds should this response be cached by HTTP proxies? Use
     ``?_ttl=0`` to disable HTTP caching entirely for this request.
 
+``?_trace=1``
+    Turns on tracing for this page: SQL queries executed during the request will
+    be gathered and included in the response, either in a new ``"_traces"`` key
+    for JSON responses or at the bottom of the page if the response is in HTML.
+
+    The structure of the data returned here should be considered highly unstable
+    and very likely to change.
+
+    Only available if the :ref:`setting_trace_debug` setting is enabled.
+
 .. _json_api_extra:
 
 Expanding JSON responses
@@ -1651,6 +1661,8 @@ The request body is always parsed as JSON, regardless of the request's ``Content
 
 The row-based write APIs can write :ref:`binary values in JSON <binary_json_format>` using Datasette's Base64 representation for BLOB data.
 
+Structured inserts, upserts, updates and deletes only support ordinary SQLite tables. Virtual tables and their internal shadow tables are rejected, including when adding rows to an existing table through the create-table API. Writes to ordinary content tables can still update full-text search indexes through configured triggers.
+
 .. _ExecuteWriteView:
 
 Executing write SQL

docs/plugins.rst

@@ -261,6 +261,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship
                 "permission_resources_sql"
             ]
         },
+        {
+            "name": "datasette.default_permissions.sqlite_statistics",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "permission_resources_sql"
+            ]
+        },
         {
             "name": "datasette.default_permissions.tokens",
             "static": false,

docs/settings.rst

@@ -71,6 +71,8 @@ Should users be able to execute arbitrary SQL queries by default?
 
 Setting this to ``off`` causes permission checks for :ref:`actions_execute_sql` to fail by default.
 
+This setting controls the ability to submit arbitrary SQL. It does not disable structured table-browsing features that use SQL generated by Datasette, such as sorting, column filters and :ref:`facets`. Use :ref:`setting_allow_facet` to control whether users can request facets.
+
 ::
 
     datasette mydatabase.db --setting default_allow_sql off
@@ -254,6 +256,8 @@ Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-ag
 
     datasette mydatabase.db --setting default_cache_ttl 60
 
+Dynamic responses for authenticated actors, requests with cookies or an ``Authorization`` header, and responses that set cookies use ``Cache-Control: private, no-store``. This takes precedence over ``default_cache_ttl`` and ``?_ttl=``, even when cache headers are otherwise disabled. Anonymous dynamic responses vary by ``Cookie`` and ``Authorization``. Static assets retain their own cache policy.
+
 .. _setting_cache_size_kb:
 
 cache_size_kb
@@ -335,6 +339,24 @@ Some examples:
 * https://latest.datasette.io/fixtures?_context=1
 * https://latest.datasette.io/fixtures/roadside_attractions?_context=1
 
+.. _setting_trace_debug:
+
+trace_debug
+~~~~~~~~~~~
+
+This setting enables appending ``?_trace=1`` to any page in order to see the SQL queries and other trace information that was used to generate that page.
+
+Enable it like this::
+
+    datasette mydatabase.db --setting trace_debug 1
+
+Some examples:
+
+* https://latest.datasette.io/?_trace=1
+* https://latest.datasette.io/fixtures/roadside_attractions?_trace=1
+
+See :ref:`internals_tracer` for details on how to hook into this mechanism as a plugin author.
+
 .. _setting_base_url:
 
 base_url

docs/telemetry_doc.py

@@ -1,37 +0,0 @@
-"""
-Render the span reference in ``internals.rst`` from
-``datasette/telemetry_registry.py``.
-
-Driven by cog, and ``cog --check docs/*.rst`` runs in CI - so adding a span
-without documenting it, or documenting one that no longer exists, is a build
-failure rather than something a reader discovers later.
-"""
-
-
-def _attribute_lines(cog, attributes):
-    if not attributes:
-        cog.out("    No attributes.\n\n")
-        return
-    cog.out("    Attributes:\n\n")
-    for attribute in attributes:
-        suffix = " *(optional)*" if attribute.optional else ""
-        cog.out(f"    - ``{attribute}``{suffix} - {attribute.description}\n")
-    cog.out("\n")
-
-
-def spans(cog):
-    from opentelemetry.trace import SpanKind
-
-    from datasette.telemetry_registry import SPANS
-
-    cog.out("\n")
-    for span in SPANS:
-        title = f"{span}*" if span.prefix else str(span)
-        cog.out(f"``{title}``\n")
-        cog.out(f"    {span.description}\n\n")
-        # INTERNAL is the default and the overwhelming majority of spans -
-        # printing it on every one would be noise. Only the exceptional case,
-        # a real database call, is worth calling out.
-        if span.kind != SpanKind.INTERNAL:
-            cog.out(f"    Kind: ``{span.kind.name}``.\n\n")
-        _attribute_lines(cog, span.attributes)

docs/testing_plugins.rst

@@ -25,7 +25,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl
         )
 
 
-This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX <https://www.python-httpx.org/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
+This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX2 <https://httpx2.pydantic.dev/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
 
 This test also uses the `pytest-asyncio <https://pypi.org/project/pytest-asyncio/>`__ package to add support for ``async def`` test functions running under pytest.
 
@@ -154,7 +154,7 @@ If you need to opt out of this behavior, add the following to your ``pytest.ini`
 Using datasette.client in tests
 -------------------------------
 
-The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client <https://www.python-httpx.org/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
+The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX2 async client <https://httpx2.pydantic.dev/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
 
 A simple test looks like this:
 
@@ -273,22 +273,22 @@ If you want to create that test database repeatedly for every individual test fu
 
 .. _testing_plugins_pytest_httpx:
 
-Testing outbound HTTP calls with pytest-httpx
----------------------------------------------
+Testing outbound HTTP calls with pytest-httpx2
+----------------------------------------------
 
 If your plugin makes outbound HTTP calls - for example datasette-auth-github or datasette-import-table - you may need to mock those HTTP requests in your tests.
 
-The `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__ package is a useful library for mocking calls. It can be tricky to use with Datasette though since it mocks all HTTPX requests, and Datasette's own testing mechanism uses HTTPX internally.
+The `pytest-httpx2 <https://pypi.org/project/pytest-httpx2/>`__ package provides a ``httpx2_mock`` fixture, built on `respx <https://lundberg.github.io/respx/>`__, for mocking outbound calls made using HTTPX2.
 
-To avoid breaking your tests, you can return ``["localhost"]`` from the ``non_mocked_hosts()`` fixture.
+Datasette's own ``datasette.client`` mechanism uses HTTPX2 internally too, but those requests are passed directly to the ASGI application rather than being sent over the network, so they are not affected by the mock.
 
-As an example, here's a very simple plugin which executes an HTTP response and returns the resulting content:
+As an example, here's a very simple plugin which executes an HTTP request and returns the resulting content:
 
 .. code-block:: python
 
     from datasette import hookimpl
     from datasette.utils.asgi import Response
-    import httpx
+    import httpx2
 
 
     @hookimpl
@@ -306,27 +306,18 @@ As an example, here's a very simple plugin which executes an HTTP response and r
             </form>""")
         vars = await request.post_vars()
         url = vars["url"]
-        return Response.text(httpx.get(url).text)
+        return Response.text(httpx2.get(url).text)
 
-Here's a test for that plugin that mocks the HTTPX outbound request:
+Here's a test for that plugin that mocks the HTTPX2 outbound request:
 
 .. code-block:: python
 
     from datasette.app import Datasette
-    import pytest
 
 
-    @pytest.fixture
-    def non_mocked_hosts():
-        # This ensures httpx-mock will not affect Datasette's own
-        # httpx calls made in the tests by datasette.client:
-        return ["localhost"]
-
-
-    async def test_outbound_http_call(httpx_mock):
-        httpx_mock.add_response(
-            url="https://www.example.com/",
-            text="Hello world",
+    async def test_outbound_http_call(httpx2_mock):
+        httpx2_mock.get("https://www.example.com/").respond(
+            text="Hello world"
         )
         datasette = Datasette([], memory=True)
         response = await datasette.client.post(
@@ -335,11 +326,13 @@ Here's a test for that plugin that mocks the HTTPX outbound request:
         )
         assert response.text == "Hello world"
 
-        outbound_request = httpx_mock.get_request()
+        outbound_request = httpx2_mock.calls.last.request
         assert (
             outbound_request.url == "https://www.example.com/"
         )
 
+If your plugin still makes its outbound calls using the original ``httpx`` library you can continue to mock those using `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__.
+
 .. _testing_plugins_register_in_test:
 
 Registering a plugin for the duration of a test

pyproject.toml

@@ -28,7 +28,7 @@ dependencies = [
     "click-default-group>=1.2.3",
     "Jinja2>=2.10.3",
     "hupper>=1.9",
-    "httpx>=0.20,<1.0",
+    "httpx2>=2.0",
     "pluggy>=1.0",
     "uvicorn>=0.29",
     "aiofiles>=0.4",
@@ -40,7 +40,6 @@ dependencies = [
     "setuptools",
     "pip",
     "pydantic>=2",
-    "opentelemetry-api>=1.37",
 ]
 
 [project.urls]
@@ -71,7 +70,6 @@ dev = [
     "cogapp>=3.3.0",
     "multipart-form-data-conformance==0.1a0",
     "ruff>=0.16.0",
-    "opentelemetry-sdk>=1.37",
     # docs
     "Sphinx==7.4.7",
     "furo==2025.9.25",

tests/conftest.py

@@ -9,7 +9,7 @@ import tempfile
 import time
 from dataclasses import dataclass
 
-import httpx
+import httpx2
 import pytest
 import pytest_asyncio
 
@@ -33,7 +33,7 @@ UNDOCUMENTED_PERMISSIONS = {
 }
 
 
-def wait_until_responds(url, timeout=5.0, client=httpx, process=None, **kwargs):
+def wait_until_responds(url, timeout=5.0, client=httpx2, process=None, **kwargs):
     start = time.time()
     while time.time() - start < timeout:
         # If the server died there is no point waiting out the timeout - fail
@@ -47,7 +47,7 @@ def wait_until_responds(url, timeout=5.0, client=httpx, process=None, **kwargs):
         try:
             client.get(url, **kwargs)
             return
-        except httpx.TransportError:
+        except httpx2.TransportError:
             time.sleep(0.1)
     raise AssertionError(f"Timed out waiting for {url} to respond")
 
@@ -58,62 +58,6 @@ def find_free_port():
         return sock.getsockname()[1]
 
 
-_otel_span_exporter = None
-
-
-@pytest.fixture(scope="session", autouse=True)
-def _otel_provider():
-    """
-    Install a real OTel SDK TracerProvider + InMemorySpanExporter exactly
-    once, before any span is ever created in this process.
-
-    This has to be session-scoped and autouse because
-    `opentelemetry.trace.set_tracer_provider()` is effectively
-    once-per-process: a second call logs a warning and is ignored. So the
-    install must happen exactly once, before anything asserts on spans.
-
-    `datasette.telemetry.tracer` is a module-level `ProxyTracer`. Once a
-    provider exists, the first span it starts resolves a concrete tracer
-    and caches it permanently. It does *not* cache the no-op tracer, so
-    any span started before this fixture runs is merely lost rather than
-    poisoning the tracer for the rest of the process. If the SDK isn't
-    installed, do nothing: core spans stay no-op `NonRecordingSpan`s and
-    the rest of the suite is unaffected.
-    """
-    global _otel_span_exporter
-    try:
-        from opentelemetry import trace as otel_trace
-        from opentelemetry.sdk.trace import TracerProvider
-        from opentelemetry.sdk.trace.export import SimpleSpanProcessor
-        from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
-            InMemorySpanExporter,
-        )
-    except ImportError:
-        return
-    exporter = InMemorySpanExporter()
-    provider = TracerProvider()
-    # SimpleSpanProcessor exports synchronously on span end - no background
-    # batching thread, so assertions immediately after a request never race.
-    provider.add_span_processor(SimpleSpanProcessor(exporter))
-    otel_trace.set_tracer_provider(provider)
-    _otel_span_exporter = exporter
-
-
-@pytest.fixture
-def otel_spans():
-    """
-    Function-scoped access to the finished-spans exporter: clears any spans
-    left over from previous tests, then yields the exporter so a test can
-    call `.get_finished_spans()` after making requests. Skips (rather than
-    fails) if the OTel SDK is not installed.
-    """
-    pytest.importorskip("opentelemetry.sdk")
-    if _otel_span_exporter is None:
-        pytest.skip("OpenTelemetry SDK provider was not installed")
-    _otel_span_exporter.clear()
-    yield _otel_span_exporter
-
-
 @pytest.fixture
 def bare_ds():
     """
@@ -224,13 +168,6 @@ def pytest_collection_modifyitems(config, items):
     move_to_front(items, "test_spatialite_error_if_attempt_to_open_spatialite")
     move_to_front(items, "test_package")
     move_to_front(items, "test_package_with_port")
-    # Same reason: this one shells out to a fresh interpreter. Late in a serial
-    # run the pytest process holds enough threads that the fork half of
-    # subprocess' fork+exec crashes the interpreter on macOS/CPython 3.13
-    # (SIGSEGV/SIGBUS inside _execute_child). Reproduces with any subprocess
-    # call placed there, on an unmodified tree - running it first avoids it.
-    move_to_front(items, "test_datasette_package_never_imports_the_sdk")
-    move_to_front(items, "test_no_provider_takes_the_fast_path")
 
 
 def move_to_front(items, test_name):
@@ -355,8 +292,8 @@ def ds_unix_domain_socket_server(tmp_path_factory):
         cwd=tempfile.gettempdir(),
     )
     # Poll until available
-    transport = httpx.HTTPTransport(uds=uds)
-    client = httpx.Client(transport=transport)
+    transport = httpx2.HTTPTransport(uds=uds)
+    client = httpx2.Client(transport=transport)
     try:
         wait_until_responds(
             "http://localhost/_memory.json", timeout=30.0, client=client
@@ -461,5 +398,6 @@ from .fixtures import (  # noqa: F401
     app_client_two_attached_databases_one_immutable,
     app_client_with_cors,
     app_client_with_dot,
+    app_client_with_trace,
     make_app_client,
 )

tests/fixtures.py

@@ -230,6 +230,12 @@ def app_client_two_attached_databases_one_immutable():
         yield client
 
 
+@pytest.fixture(scope="session")
+def app_client_with_trace():
+    with make_app_client(settings={"trace_debug": True}, is_immutable=True) as client:
+        yield client
+
+
 @pytest.fixture(scope="session")
 def app_client_shorter_time_limit():
     with make_app_client(20) as client:

tests/plugins/my_plugin.py

@@ -3,7 +3,7 @@ import base64
 import json
 import urllib.parse
 
-from datasette import hookimpl
+from datasette import hookimpl, tracer
 from datasette.facets import Facet
 from datasette.permissions import Action
 from datasette.resources import DatabaseResource
@@ -278,10 +278,11 @@ def register_routes():
 
     async def parallel_queries(datasette):
         db = datasette.get_database()
-        one, two = await asyncio.gather(
-            db.execute("select coalesce(sleep(0.1), 1)"),
-            db.execute("select coalesce(sleep(0.1), 2)"),
-        )
+        with tracer.trace_child_tasks():
+            one, two = await asyncio.gather(
+                db.execute("select coalesce(sleep(0.1), 1)"),
+                db.execute("select coalesce(sleep(0.1), 2)"),
+            )
         return Response.json({"one": one.single_value(), "two": two.single_value()})
 
     return [

tests/test_api.py

@@ -5,6 +5,7 @@ import pytest
 
 from datasette.app import Datasette
 from datasette.plugins import DEFAULT_PLUGINS
+from datasette.resources import DatabaseResource, TableResource
 from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode
 from datasette.utils.sqlite import sqlite_version
 from datasette.version import __version__
@@ -101,14 +102,11 @@ async def test_database_page(ds_client):
         "tags",
     }
 
-    # Expected hidden tables
+    # The external-content index is visible, but its shadow tables need a
+    # second dependency hop and are excluded by the one-hop permission policy.
     expected_hidden_tables = {
         "no_primary_key",
         "searchable_fts",
-        "searchable_fts_config",
-        "searchable_fts_data",
-        "searchable_fts_docsize",
-        "searchable_fts_idx",
     }
 
     # Verify all expected tables exist
@@ -458,6 +456,67 @@ async def test_row_foreign_key_tables(ds_client):
     ]
 
 
+@pytest.mark.asyncio
+async def test_row_foreign_key_tables_omit_denied_tables(request):
+    actor = {"id": "reader"}
+    ds = Datasette(
+        memory=True,
+        default_deny=True,
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "parents": {"permissions": {"view-table": True}},
+                        "private_children": {"permissions": {"view-table": False}},
+                    }
+                }
+            }
+        },
+    )
+    request.addfinalizer(ds.close)
+    db = ds.add_memory_database("fk_count_leak", name="data")
+    await db.execute_write("create table parents (id integer primary key, name text)")
+    await db.execute_write("""
+        create table private_children (
+            id integer primary key,
+            parent_id integer references parents(id)
+        )
+    """)
+    await db.execute_write("insert into parents values (1, 'Public parent')")
+    await db.execute_write("""
+        insert into private_children (id, parent_id) values
+            (1, 1),
+            (2, 1),
+            (3, 1)
+    """)
+    await ds.invoke_startup()
+
+    parent = TableResource(database="data", table="parents")
+    private_children = TableResource(database="data", table="private_children")
+    assert await ds.allowed(action="view-table", resource=parent, actor=actor)
+    assert not await ds.allowed(
+        action="view-table", resource=private_children, actor=actor
+    )
+    assert not await ds.allowed(
+        action="execute-sql",
+        resource=DatabaseResource(database="data"),
+        actor=actor,
+    )
+
+    direct_child = await ds.client.get("/data/private_children.json", actor=actor)
+    assert direct_child.status_code == 403
+    parent_response = await ds.client.get(
+        "/data/parents/1.json?_extra=foreign_key_tables", actor=actor
+    )
+    assert parent_response.status_code == 200
+
+    foreign_key_tables = parent_response.json().get("foreign_key_tables", [])
+    assert foreign_key_tables == [], (
+        "denied child table name, foreign-key column, and row count disclosed: "
+        f"{foreign_key_tables}"
+    )
+
+
 @pytest.mark.asyncio
 async def test_row_extras(ds_client):
     response = await ds_client.get(
@@ -706,6 +765,7 @@ async def test_settings_json(ds_client):
         "truncate_cells_html": 2048,
         "force_https_urls": False,
         "template_debug": False,
+        "trace_debug": False,
         "base_url": "/",
     }
 
@@ -893,10 +953,7 @@ async def test_hidden_sqlite_stat1_table():
     await db.execute_write("analyze")
     data = (await ds.client.get("/db.json?_show_hidden=1")).json()
     tables = [(t["name"], t["hidden"]) for t in data["tables"]]
-    assert tables in (
-        [("normal", False), ("sqlite_stat1", True)],
-        [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)],
-    )
+    assert tables == [("normal", False)]
 
 
 @pytest.mark.asyncio

tests/test_api_write.py

@@ -68,6 +68,82 @@ BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"}
 BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}'
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_fallback", (False, True))
+@pytest.mark.parametrize(
+    "operation", ("insert", "upsert", "update", "delete", "create", "create_uppercase")
+)
+@pytest.mark.parametrize(
+    "module,definition,values,shadow_suffix",
+    (
+        ("fts5", "body", "'original'", "_content"),
+        ("fts4", "body", "'original'", "_content"),
+        ("rtree", "id, minx, maxx", "1, 0, 1", "_rowid"),
+    ),
+)
+@pytest.mark.parametrize("shadow", (False, True))
+async def test_structured_writes_require_ordinary_tables(
+    ds_write,
+    monkeypatch,
+    use_fallback,
+    operation,
+    module,
+    definition,
+    values,
+    shadow_suffix,
+    shadow,
+):
+    if use_fallback:
+        monkeypatch.setattr("datasette.utils.sqlite.supports_table_list", lambda: False)
+    db = ds_write.get_database("data")
+    await db.execute_write(f"create virtual table indexed using {module}({definition})")
+    await db.execute_write(f"insert into indexed values ({values})")
+    table = "indexed" + (shadow_suffix if shadow else "")
+    row = (await db.execute(f"select rowid, * from {escape_sqlite(table)}")).dicts()[0]
+    pks = await db.primary_keys(table)
+    pk_value = row[pks[0] if pks else "rowid"]
+    before = await db.execute_fn(lambda conn: list(conn.iterdump()))
+
+    if operation in ("create", "create_uppercase"):
+        path = "/data/-/create"
+        body = {
+            "table": table.upper() if operation == "create_uppercase" else table,
+            "rows": [row],
+        }
+    elif operation in ("update", "delete"):
+        path = f"/data/{table}/{pk_value}/-/{operation}"
+        body = {"update": row} if operation == "update" else {}
+    else:
+        path = f"/data/{table}/-/{operation}"
+        body = {"rows": [row]}
+    response = await ds_write.client.post(
+        path, json=body, headers=_headers(write_token(ds_write))
+    )
+    assert response.status_code == 400, response.text
+    assert response.json()["errors"] == ["Structured writes require an ordinary table"]
+    assert await db.execute_fn(lambda conn: list(conn.iterdump())) == before
+
+
+@pytest.mark.asyncio
+async def test_structured_writes_to_content_table_maintain_fts(ds_write):
+    db = ds_write.get_database("data")
+    await db.execute_write_fn(
+        lambda conn: sqlite_utils.Database(conn)["docs"].enable_fts(
+            ["title"], create_triggers=True
+        )
+    )
+    response = await ds_write.client.post(
+        "/data/docs/-/insert",
+        json={"row": {"id": 1, "title": "ordinary content"}},
+        headers=_headers(write_token(ds_write)),
+    )
+    assert response.status_code == 201, response.text
+    matches = await db.execute(
+        "select rowid from docs_fts where docs_fts match ?", ["ordinary"]
+    )
+    assert [row[0] for row in matches.rows] == [1]
+
+
 @pytest.mark.asyncio
 async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write):
     token = write_token(ds_write)
@@ -1296,7 +1372,7 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w
 
 @pytest.mark.asyncio
 async def test_foreign_key_suggestions(ds_write):
-    token = write_token(ds_write, permissions=["at"])
+    token = write_token(ds_write, permissions=["alter-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
     await db.execute_write("insert into owners (id) values (1), (2), (3)")
@@ -1362,7 +1438,7 @@ async def test_foreign_key_suggestions_permission_denied(ds_write):
 
 @pytest.mark.asyncio
 async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch):
-    token = write_token(ds_write, permissions=["at"])
+    token = write_token(ds_write, permissions=["alter-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
 
@@ -1393,7 +1469,7 @@ async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch):
 
 @pytest.mark.asyncio
 async def test_foreign_key_targets(ds_write):
-    token = write_token(ds_write, permissions=["ct"])
+    token = write_token(ds_write, permissions=["create-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
     await db.execute_write("create table categories (slug varchar(30) primary key)")
@@ -2745,3 +2821,119 @@ async def test_create_using_alter_against_existing_table(
         insert_rows_event = ds_write._tracked_events[1]
         assert insert_rows_event.name == "insert-rows"
         assert insert_rows_event.num_rows == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("denied_action", "request_body"),
+    (
+        (
+            "insert-row",
+            {
+                "table": "salaries",
+                "rows": [{"id": 9, "note": "INJ-VIA-CREATE"}],
+            },
+        ),
+        (
+            "update-row",
+            {
+                "table": "salaries",
+                "rows": [{"id": 1, "note": "REPLACED"}],
+                "pk": "id",
+                "replace": True,
+            },
+        ),
+        (
+            "alter-table",
+            {
+                "table": "salaries",
+                "rows": [{"id": 9, "note": "INSERTED", "extra": "NEW"}],
+                "alter": True,
+            },
+        ),
+    ),
+)
+async def test_create_table_existing_table_respects_table_level_denial(
+    denied_action, request_body
+):
+    # GHSA-53fc-rhfg-h7qp issue 2: POST /db/-/create against an existing table
+    # inserts rows into it, so insert-row (and update-row / alter-table) must be
+    # checked against the TableResource, not just the DatabaseResource.
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                # id=editor user has each permission at the database level, but
+                # the selected action is explicitly denied on the salaries table
+                "data": {
+                    "permissions": {
+                        "create-table": {"id": "editor"},
+                        "insert-row": {"id": "editor"},
+                        "update-row": {"id": "editor"},
+                        "alter-table": {"id": "editor"},
+                    },
+                    "tables": {
+                        "salaries": {"permissions": {denied_action: False}},
+                    },
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database(
+        f"create_table_existing_table_denied_{denied_action}", name="data"
+    )
+    await db.execute_write("create table salaries (id integer primary key, note text)")
+    await db.execute_write("insert into salaries values (1, 'TOPSECRET-A')")
+    await ds.invoke_startup()
+
+    if denied_action == "insert-row":
+        # Sanity: direct insert into salaries is denied for this actor
+        direct = await ds.client.post(
+            "/data/salaries/-/insert",
+            actor={"id": "editor"},
+            json={"row": {"id": 9, "note": "INJ-DIRECT"}},
+        )
+        assert direct.status_code == 403
+
+    response = await ds.client.post(
+        "/data/-/create",
+        actor={"id": "editor"},
+        json=request_body,
+    )
+    assert response.status_code == 403, response.json()
+    assert response.json()["errors"] == [f"Permission denied: need {denied_action}"]
+    rows = (await db.execute("select id, note from salaries order by id")).rows
+    assert [tuple(r) for r in rows] == [(1, "TOPSECRET-A")]
+    assert await db.table_columns("salaries") == ["id", "note"]
+
+
+@pytest.mark.asyncio
+async def test_create_table_respects_predeclared_table_level_denial():
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                "data": {
+                    "permissions": {
+                        "create-table": {"id": "editor"},
+                        "insert-row": {"id": "editor"},
+                    },
+                    "tables": {
+                        "planned_table": {"permissions": {"insert-row": False}},
+                    },
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database("create_table_predeclared_denial", name="data")
+    await ds.invoke_startup()
+
+    response = await ds.client.post(
+        "/data/-/create",
+        actor={"id": "editor"},
+        json={"table": "planned_table", "rows": [{"id": 1}]},
+    )
+
+    assert response.status_code == 403, response.json()
+    assert response.json()["errors"] == ["Permission denied: need insert-row"]
+    assert not await db.table_exists("planned_table")

tests/test_auth.py

@@ -1,4 +1,5 @@
 import time
+from unittest.mock import AsyncMock
 
 import pytest
 from bs4 import BeautifulSoup as Soup
@@ -237,6 +238,35 @@ def test_auth_create_token(
         assert response3.json["actor"]["id"] == "test"
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["GET", "POST"])
+@pytest.mark.parametrize(
+    "restrictions",
+    [
+        {},
+        {"a": ["vi"]},
+        {"d": {"db": ["vd"]}},
+        {"r": {"db": {"t1": ["vt"]}}},
+    ],
+    ids=["empty", "instance", "database", "table"],
+)
+async def test_auth_create_token_not_allowed_for_restricted_actors(
+    bare_ds, monkeypatch, method, restrictions
+):
+    create_token = AsyncMock()
+    monkeypatch.setattr(bare_ds, "create_token", create_token)
+
+    response = await bare_ds.client.request(
+        method,
+        "/-/create-token",
+        actor={"id": "test", "_r": restrictions},
+    )
+
+    assert response.status_code == 403
+    assert "Restricted actors cannot create API tokens" in response.text
+    create_token.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_auth_create_token_not_allowed_for_tokens(ds_client):
     ds_tok = ds_client.ds.sign(
@@ -524,3 +554,25 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client):
         )
         is not True
     ), "Root without root_enabled should not automatically get set-column-type"
+
+
+@pytest.mark.parametrize("expire_after", (1, 300, 3600, 30 * 24 * 60 * 60))
+def test_set_actor_cookie_honours_expire_after(expire_after):
+    # GHSA-53fc-rhfg-h7qp issue 4: expire_after is documented as a number of
+    # seconds, but every value was being replaced with 24 hours.
+    from datasette.app import Datasette
+    from datasette.utils.asgi import Response
+
+    ds = Datasette(memory=True)
+    response = Response.text("")
+    before = int(time.time())
+    ds.set_actor_cookie(response, {"id": "test"}, expire_after=expire_after)
+    after = int(time.time())
+
+    (header,) = response._set_cookie_headers
+    assert header.startswith("ds_actor=")
+    value = header[len("ds_actor=") :].split(";", 1)[0]
+    data = ds.unsign(value, "actor")
+    assert data["a"] == {"id": "test"}
+    expires_at = baseconv.base62.decode(data["e"])
+    assert before + expire_after <= expires_at <= after + expire_after

tests/test_cli_serve_server.py

@@ -1,13 +1,13 @@
 import socket
 import time
 
-import httpx
+import httpx2
 import pytest
 
 
 @pytest.mark.serial
 def test_serve_localhost_http(ds_localhost_http_server):
-    response = httpx.get("http://localhost:8041/_memory.json")
+    response = httpx2.get("http://localhost:8041/_memory.json")
     assert {
         "database": "_memory",
         "path": "/_memory",
@@ -21,8 +21,8 @@ def test_serve_localhost_http(ds_localhost_http_server):
 )
 def test_serve_unix_domain_socket(ds_unix_domain_socket_server):
     _, uds = ds_unix_domain_socket_server
-    transport = httpx.HTTPTransport(uds=uds)
-    client = httpx.Client(transport=transport)
+    transport = httpx2.HTTPTransport(uds=uds)
+    client = httpx2.Client(transport=transport)
     response = client.get("http://localhost/_memory.json")
     assert {
         "database": "_memory",
@@ -97,7 +97,7 @@ def test_startup_hook_background_task_runs_on_serving_loop(serve_with_plugins):
     deadline = time.time() + 3.0
     payload = {}
     while time.time() < deadline:
-        payload = httpx.get(
+        payload = httpx2.get(
             f"http://127.0.0.1:{port}/-/marker-task-ran", timeout=1.0
         ).json()
         if payload["marker_task_ran"]:

tests/test_csv.py

@@ -1,9 +1,9 @@
 import urllib.parse
 
 import pytest
+from bs4 import BeautifulSoup as Soup
 
 from datasette.app import Datasette
-from datasette.telemetry_registry import DB_QUERY, DB_QUERY_TEXT
 
 EXPECTED_TABLE_CSV = """id,content
 1,hello
@@ -229,43 +229,24 @@ async def test_table_csv_stream(ds_client):
     assert len([b for b in response.content.split(b"\r\n") if b]) == 1002
 
 
-def db_query_texts(otel_spans):
-    "Every db.query.text recorded by a db.query span since the exporter was cleared."
-    return [
-        span.attributes.get(DB_QUERY_TEXT, "")
-        for span in otel_spans.get_finished_spans()
-        if span.name == DB_QUERY
-    ]
+def test_csv_trace(app_client_with_trace):
+    response = app_client_with_trace.get("/fixtures/simple_primary_key.csv?_trace=1")
+    assert response.headers["content-type"] == "text/html; charset=utf-8"
+    soup = Soup(response.text, "html.parser")
+    assert (
+        soup.find("textarea").text
+        == "id,content\r\n1,hello\r\n2,world\r\n3,\r\n4,RENDER_CELL_DEMO\r\n5,RENDER_CELL_ASYNC\r\n"
+    )
+    assert "select id, content from simple_primary_key" in soup.find("pre").text
 
 
-# Both faceting and facet suggestion aggregate with a named count: facet
-# results use "count(*) as count", suggestions use "count(*) as n". Matching
-# on those rather than on a whole query string, because the surrounding SQL
-# has been rewritten before - the previous version of this test looked for
-# "select content, count(*) as n", which facet suggestion stopped emitting
-# when it moved to a "with limited as (...)" CTE, leaving the assertion
-# unable to fail.
-FACET_QUERY_MARKERS = ("count(*) as n", "count(*) as count")
+def test_table_csv_stream_does_not_calculate_facets(app_client_with_trace):
+    response = app_client_with_trace.get("/fixtures/simple_primary_key.csv?_trace=1")
+    soup = Soup(response.text, "html.parser")
+    assert "select content, count(*) as n" not in soup.find("pre").text
 
 
-@pytest.mark.asyncio
-async def test_table_csv_stream_does_not_calculate_facets(ds_client, otel_spans):
-    response = await ds_client.get("/fixtures/simple_primary_key.csv")
-    assert response.status_code == 200
-    queries = db_query_texts(otel_spans)
-    # Guard: without this, a change that stopped the CSV route running any
-    # query at all - or that broke span capture - would leave the real
-    # assertion below trivially true.
-    assert any("from simple_primary_key" in q for q in queries), queries
-    assert not any(
-        marker in query for query in queries for marker in FACET_QUERY_MARKERS
-    ), queries
-
-
-@pytest.mark.asyncio
-async def test_table_csv_stream_does_not_calculate_counts(ds_client, otel_spans):
-    response = await ds_client.get("/fixtures/simple_primary_key.csv")
-    assert response.status_code == 200
-    queries = db_query_texts(otel_spans)
-    assert any("from simple_primary_key" in q for q in queries), queries
-    assert not any("select count(*)" in q for q in queries), queries
+def test_table_csv_stream_does_not_calculate_counts(app_client_with_trace):
+    response = app_client_with_trace.get("/fixtures/simple_primary_key.csv?_trace=1")
+    soup = Soup(response.text, "html.parser")
+    assert "select count(*)" not in soup.find("pre").text

tests/test_fts_permissions.py

@@ -0,0 +1,412 @@
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.permissions import Action, PermissionSQL, _permission_check_cache
+from datasette.resources import DatabaseResource, TableResource
+from datasette.utils.sqlite import sqlite3, sqlite_derived_table_dependencies
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ["fts4", "fts5"])
+@pytest.mark.parametrize("actor", [None, {"id": "root"}], ids=["anonymous", "root"])
+async def test_derived_permissions_allow_one_hop_but_deny_nested_sources(
+    fts_module, actor
+):
+    class InspectPlugin:
+        @hookimpl
+        def register_actions(self):
+            return [
+                Action(
+                    name="inspect-derived",
+                    description="Inspect a table",
+                    resource_class=TableResource,
+                    also_requires="view-table",
+                )
+            ]
+
+        @hookimpl
+        def permission_resources_sql(self, action):
+            if action == "inspect-derived":
+                return PermissionSQL(
+                    sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'inspect allowed' AS reason"
+                )
+
+    ds = Datasette(memory=True)
+    ds.pm.register(InspectPlugin(), name="inspect-derived-test")
+    db = ds.add_memory_database(
+        f"derived_one_hop_{fts_module}_{actor is not None}", name="data"
+    )
+    await db.execute_write("create table Documents (body text)")
+    await db.execute_write(
+        f"create virtual table Search using {fts_module}(body, content='Documents')"
+    )
+    await db.execute_write(
+        f"create virtual table Nested using {fts_module}(body, content='sEaRcH')"
+    )
+    await ds.invoke_startup()
+    token = _permission_check_cache.set({})
+    try:
+        # Both direct permissions are allowed, but a derived source makes its
+        # dependent unavailable even to an actor who can view the whole chain.
+        # Check and cache Search first so its cached grant cannot grant Nested.
+        for table, expected in (
+            ("Documents", True),
+            ("Search", True),
+            ("Nested", False),
+            ("Search_docsize", False),
+        ):
+            for spelling in (table, table.upper(), table.lower()):
+                assert await ds.allowed_many(
+                    actions=["view-table", "inspect-derived"],
+                    resource=TableResource("data", spelling),
+                    actor=actor,
+                ) == {"view-table": expected, "inspect-derived": expected}
+
+        page = await ds.allowed_resources(
+            "view-table", actor, parent="data", include_is_private=True, limit=1000
+        )
+        allowed = {resource.child for resource in page.resources}
+        assert {"Documents", "Search"}.issubset(allowed)
+        assert "Nested" not in allowed
+        assert "Search_docsize" not in allowed
+    finally:
+        _permission_check_cache.reset(token)
+        ds.pm.unregister(name="inspect-derived-test")
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("listing", [False, True], ids=["individual", "listing"])
+async def test_derived_permission_discovery_error_is_retried(monkeypatch, listing):
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(f"derived_discovery_error_{listing}", name="data")
+    await db.execute_write("create table documents (id integer primary key)")
+    await ds.invoke_startup()
+
+    class UnavailableSchema:
+        def execute(self, sql):
+            raise sqlite3.DatabaseError("schema temporarily unavailable")
+
+    async def check():
+        if listing:
+            return await ds.allowed_resources("view-table", parent="data")
+        return await ds.allowed(
+            action="view-table", resource=TableResource("data", "documents")
+        )
+
+    token = _permission_check_cache.set({})
+    try:
+        with monkeypatch.context() as patch:
+            patch.setattr(
+                "datasette.database.sqlite_derived_table_dependencies",
+                lambda conn: sqlite_derived_table_dependencies(UnavailableSchema()),
+            )
+            with pytest.raises(sqlite3.DatabaseError, match="schema temporarily"):
+                await check()
+
+        # Failed discovery must not cache an empty map or a permission grant.
+        assert db._cached_derived_table_dependencies is None
+        assert not _permission_check_cache.get()
+        result = await check()
+        if listing:
+            assert [resource.child for resource in result.resources] == ["documents"]
+        else:
+            assert result is True
+        assert db._cached_derived_table_dependencies is not None
+    finally:
+        _permission_check_cache.reset(token)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ("fts4", "fts5"))
+async def test_external_content_fts_inherits_content_table_view_permission(fts_module):
+    actor = {"id": "reader"}
+    secret_marker = "ISSUE_17_EXTERNAL_CONTENT_FTS_SECRET"
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": {"id": "reader"},
+                "view-database": {"id": "reader"},
+                "view-table": {"id": "reader"},
+                "execute-sql": {"id": "nobody"},
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": False}},
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database(f"issue_17_{fts_module}_permissions", name="data")
+    await db.execute_write("create table secret (id integer primary key, body text)")
+    await db.execute_write(
+        "insert into secret (body) values (?)",
+        [secret_marker],
+    )
+    fts_options = "body, content='secret'"
+    if fts_module == "fts5":
+        fts_options += ", content_rowid='id'"
+    await db.execute_write(
+        f"create virtual table secret_fts using {fts_module}({fts_options})"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        assert "secret_fts" in await db.hidden_table_names()
+        assert (
+            await ds.allowed(
+                action="execute-sql",
+                resource=DatabaseResource("data"),
+                actor=actor,
+            )
+            is False
+        )
+
+        direct = await ds.client.get("/data/secret.json", actor=actor)
+        assert direct.status_code == 403
+
+        companion = await ds.client.get(
+            "/data/secret_fts.json?_shape=array",
+            actor=actor,
+        )
+        assert companion.status_code in (403, 404), (
+            "An automatically hidden external-content FTS table must inherit "
+            "the content table's view denial or be unavailable: "
+            f"{companion.text}"
+        )
+        assert secret_marker not in companion.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ("fts4", "fts5"))
+@pytest.mark.parametrize("contentless", (False, True), ids=("internal", "contentless"))
+async def test_fts_shadow_tables_inherit_logical_table_view_permission(
+    fts_module, contentless
+):
+    table_config = {
+        "secret_fts": {"permissions": {"view-table": False}},
+        # An explicit allow on one implementation table must not override
+        # the logical FTS table's denial.
+        "secret_fts_docsize": {"permissions": {"view-table": True}},
+    }
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+                "view-table": True,
+                "execute-sql": False,
+            },
+            "databases": {"data": {"tables": table_config}},
+        },
+    )
+    db = ds.add_memory_database(
+        f"issue_17_{fts_module}_{'contentless' if contentless else 'internal'}",
+        name="data",
+    )
+    options = "body, content=''" if contentless else "body"
+    await db.execute_write(
+        f"create virtual table secret_fts using {fts_module}({options})"
+    )
+    await db.execute_write(
+        "insert into secret_fts(rowid, body) values (1, 'ISSUE_17_SHADOW_SECRET')"
+    )
+    await ds.invoke_startup()
+
+    try:
+        dependencies = await db.derived_table_dependencies()
+        shadow_tables = sorted(
+            table for table, source in dependencies.items() if source == "secret_fts"
+        )
+        assert shadow_tables
+        assert "secret_fts_docsize" in shadow_tables
+
+        for shadow_table in shadow_tables:
+            assert (
+                await ds.allowed(
+                    action="view-table",
+                    resource=TableResource("data", shadow_table),
+                )
+                is False
+            )
+            response = await ds.client.get(f"/data/{shadow_table}.json?_shape=array")
+            assert response.status_code == 403
+            assert "ISSUE_17_SHADOW_SECRET" not in response.text
+
+        allowed = await ds.allowed_resources("view-table", parent="data", limit=1000)
+        allowed_names = {resource.child for resource in allowed.resources}
+        assert not set(shadow_tables).intersection(allowed_names)
+
+        database_json = await ds.client.get("/data.json")
+        assert database_json.status_code == 200
+        for shadow_table in shadow_tables:
+            assert shadow_table not in database_json.text
+
+        schema_json = await ds.client.get("/data/-/schema.json")
+        assert schema_json.status_code == 200
+        for shadow_table in shadow_tables:
+            assert shadow_table not in schema_json.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "content_allowed,companion_allowed,expected",
+    (
+        (False, True, False),
+        (True, False, False),
+        (True, True, True),
+    ),
+)
+async def test_external_content_and_companion_permissions_are_both_required(
+    content_allowed, companion_allowed, expected
+):
+    ds = Datasette(
+        memory=True,
+        default_deny=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": content_allowed}},
+                        "secret_fts": {
+                            "permissions": {"view-table": companion_allowed}
+                        },
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database(
+        f"issue_17_explicit_{int(content_allowed)}_{int(companion_allowed)}",
+        name="data",
+    )
+    await db.execute_write("create table secret(id integer primary key, body text)")
+    await db.execute_write("insert into secret(body) values ('ISSUE_17_MATRIX_SECRET')")
+    await db.execute_write(
+        "create virtual table secret_fts using fts5("
+        "body, content='secret', content_rowid='id')"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        assert (
+            await ds.allowed(
+                action="view-table",
+                resource=TableResource("data", "secret_fts"),
+            )
+            is expected
+        )
+        response = await ds.client.get("/data/secret_fts.json?_shape=array")
+        assert response.status_code == (200 if expected else 403)
+        if not expected:
+            assert "ISSUE_17_MATRIX_SECRET" not in response.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_derived_tables_propagate_private_flag_and_route_permissions():
+    actor = {"id": "reader"}
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+                "view-table": True,
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": {"id": "reader"}}}
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database("issue_17_private_flag", name="data")
+    await db.execute_write("create table secret(id integer primary key, body text)")
+    await db.execute_write("insert into secret(body) values ('PRIVATE')")
+    await db.execute_write(
+        "create virtual table secret_fts using fts5("
+        "body, content='secret', content_rowid='id')"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        actor_page = await ds.allowed_resources(
+            "view-table", actor, parent="data", include_is_private=True, limit=1000
+        )
+        actor_resources = {
+            resource.child: resource for resource in actor_page.resources
+        }
+        derived_names = set(await db.derived_table_dependencies())
+        assert "secret_fts" in actor_resources
+        assert actor_resources["secret_fts"].private
+        # Shadow tables depend on the already-derived external-content FTS
+        # table, so they remain unavailable even to the permitted reader.
+        assert not (derived_names - {"secret_fts"}).intersection(actor_resources)
+
+        anonymous_page = await ds.allowed_resources(
+            "view-table", parent="data", limit=1000
+        )
+        anonymous_names = {resource.child for resource in anonymous_page.resources}
+        assert not derived_names.intersection(anonymous_names)
+
+        for path in (
+            "/data/secret_fts.json?_facet=body",
+            "/data/secret_fts.csv",
+            "/data/secret_fts/-/autocomplete?q=PRIVATE",
+            "/data/secret_fts/-/schema.json",
+        ):
+            denied = await ds.client.get(path)
+            assert denied.status_code == 403
+            allowed = await ds.client.get(path, actor=actor)
+            assert allowed.status_code == 200
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_cyclic_derived_table_dependencies_fail_closed():
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database("issue_17_cycle", name="data")
+    await db.execute_write(
+        "create virtual table first_fts using fts5(body, content='second_fts')"
+    )
+    await db.execute_write(
+        "create virtual table second_fts using fts5(body, content='first_fts')"
+    )
+    await ds.invoke_startup()
+
+    try:
+        for table in ("first_fts", "second_fts"):
+            assert (
+                await ds.allowed(
+                    action="view-table", resource=TableResource("data", table)
+                )
+                is False
+            )
+
+        page = await ds.allowed_resources("view-table", parent="data", limit=1000)
+        allowed_names = {resource.child for resource in page.resources}
+        assert "first_fts" not in allowed_names
+        assert "second_fts" not in allowed_names
+    finally:
+        ds.close()

tests/test_html.py

@@ -36,8 +36,10 @@ def test_homepage(app_client_two_attached_databases):
     h2 = soup.select("h2")[0]
     assert "extra database" == h2.text.strip()
     counts_p, links_p = h2.find_all_next("p")[:2]
+    # Shadow tables of the external-content index are denied, so they do not
+    # contribute to the table or row totals.
     assert (
-        "2 rows in 1 table, 5 rows in 4 hidden tables, 1 view" == counts_p.text.strip()
+        "2 rows in 1 table, 2 rows in 1 hidden table, 1 view" == counts_p.text.strip()
     )
     # We should only show visible, not hidden tables here:
     table_links = [
@@ -1141,13 +1143,8 @@ async def test_navigation_menu_links(
 
 
 @pytest.mark.asyncio
-async def test_query_page_escapes_sql(ds_client):
-    # This was previously test_trace_correctly_escaped, which appended
-    # ?_trace=1. It never exercised the tracer - ds_client has no trace_debug -
-    # so what it actually covered was the query page echoing user-supplied SQL
-    # back into HTML. That page is the subject of two historical reflected-XSS
-    # advisories (issue 1360), so the coverage is kept now the tracer is gone.
-    response = await ds_client.get("/fixtures/-/query?sql=select+'<h1>Hello'")
+async def test_trace_correctly_escaped(ds_client):
+    response = await ds_client.get("/fixtures/-/query?sql=select+'<h1>Hello'&_trace=1")
     assert "select '<h1>Hello" not in response.text
     assert "select &#39;&lt;h1&gt;Hello" in response.text
 

tests/test_http_span.py

@@ -1,855 +0,0 @@
-"""
-The HTTP request span.
-
-`tests/test_telemetry_registry.py` already pins the span's name shape, kind
-and attribute keys against literals, so this file deliberately does not
-repeat that. What it covers is the properties of the middleware and of the
-router's `http.route` enrichment that the registry conformance test
-structurally cannot see:
-
-- **where the middleware sits.** Outermost is the entire point - moving it
-  inside the plugin `asgi_wrapper()` loop leaves plugin middleware creating
-  orphan root traces, which is the problem this span exists to fix, and every
-  attribute assertion still passes.
-- **which span the route lands on**, which only diverges once something else
-  has made a span current.
-- **method clamping**, which a workload of ordinary GETs can never exercise.
-- **the query string never being recorded**, which only fails if a request
-  actually carries one.
-- **the span outliving a streamed response body**, which only a paging export
-  can distinguish from ending far too early.
-"""
-
-import asyncio
-import itertools
-import json
-import subprocess
-import sys
-import textwrap
-import time
-
-import pytest
-import pytest_asyncio
-
-pytest.importorskip("opentelemetry.sdk")
-
-from opentelemetry.trace import (
-    NonRecordingSpan,
-    SpanContext,
-    SpanKind,
-    StatusCode,
-    TraceFlags,
-)
-
-from datasette import hookimpl
-from datasette.app import Datasette
-from datasette.telemetry import (
-    REQUEST_SPAN_SCOPE_KEY,
-    TelemetryMiddleware,
-    request_span,
-    tracer,
-)
-from datasette.utils import resolve_routes
-
-# Named in-memory databases are shared-cache: two Datasette instances given
-# the same name share one SQLite database and the second `create table`
-# fails.
-_names = itertools.count()
-
-
-PLUGIN_MIDDLEWARE_SPAN = "test.plugin.middleware"
-
-
-class _MiddlewarePlugin:
-    "A plugin asgi_wrapper() that creates a span, standing in for a real one."
-
-    __name__ = "HttpSpanMiddlewarePlugin"
-
-    @hookimpl
-    def asgi_wrapper(self, datasette):
-        def wrap(app):
-            async def wrapped(scope, receive, send):
-                with tracer.start_as_current_span(PLUGIN_MIDDLEWARE_SPAN):
-                    await app(scope, receive, send)
-
-            return wrapped
-
-        return wrap
-
-
-class _RaisingMiddlewarePlugin:
-    """
-    A plugin asgi_wrapper() that raises.
-
-    `route_path` converts almost every exception into a 500 itself, so an
-    exception escaping into the request span is only reachable from *outside*
-    the router - a plugin wrapper, or a failure inside the 500 handler.
-    """
-
-    __name__ = "HttpSpanRaisingMiddlewarePlugin"
-
-    def __init__(self, call_app_first):
-        self.call_app_first = call_app_first
-
-    @hookimpl
-    def asgi_wrapper(self, datasette):
-        call_app_first = self.call_app_first
-
-        def wrap(app):
-            async def wrapped(scope, receive, send):
-                if call_app_first:
-                    await app(scope, receive, send)
-                raise RuntimeError("wrapper exploded")
-
-            return wrapped
-
-        return wrap
-
-
-class _BoomPlugin:
-    "A route that raises, which route_path turns into a 500."
-
-    __name__ = "HttpSpanBoomPlugin"
-
-    @hookimpl
-    def register_routes(self):
-        return [(r"^/-/http-span-boom$", lambda: 1 / 0)]
-
-
-@pytest_asyncio.fixture
-async def ds():
-    name = f"httpspan{next(_names)}"
-    instance = Datasette(memory=True)
-    instance.add_memory_database(name)
-    await instance.invoke_startup()
-    await instance.get_database(name).execute_write(
-        "create table t (id integer primary key, v text)"
-    )
-    instance.db_name = name
-    try:
-        yield instance
-    finally:
-        instance.close()
-
-
-@pytest_asyncio.fixture
-async def ds_paging():
-    """
-    An instance whose table is bigger than `max_returned_rows`.
-
-    That is what makes `?_stream=1` genuinely page: `stream_csv` loops calling
-    `fetch_data` for each page *inside* the response body send, so the trace
-    contains `db.query` spans that start after the response has begun. On a
-    table that fits in one page every query finishes before the body starts
-    and the span-covers-the-body assertion cannot fail.
-    """
-    name = f"httpspanpaging{next(_names)}"
-    # Both settings matter. `?_stream=1` forces `_size=max`, which is
-    # `max_returned_rows` - so lowering only that gives one page of five rows
-    # and no `next` token, and the export never loops.
-    instance = Datasette(
-        memory=True, settings={"max_returned_rows": 5, "default_page_size": 3}
-    )
-    instance.add_memory_database(name)
-    await instance.invoke_startup()
-    db = instance.get_database(name)
-    await db.execute_write("create table t (id integer primary key, v text)")
-    await db.execute_write_many(
-        "insert into t (id, v) values (?, ?)", [[i, f"v{i}"] for i in range(40)]
-    )
-    instance.db_name = name
-    try:
-        yield instance
-    finally:
-        instance.close()
-
-
-def _server_spans(otel_spans):
-    return [
-        span for span in otel_spans.get_finished_spans() if span.kind is SpanKind.SERVER
-    ]
-
-
-def _route_for(ds, path):
-    "The compiled pattern Datasette's own router resolves `path` to."
-    match, _view = resolve_routes(ds._routes(), path)
-    assert match is not None, f"{path} matches no route"
-    return match.re.pattern
-
-
-@pytest.mark.asyncio
-async def test_plugin_asgi_wrapper_middleware_runs_inside_the_request_span(
-    ds, otel_spans
-):
-    """
-    The placement check.
-
-    A span created by a plugin `asgi_wrapper()` must be a *child* of the
-    request span. If the middleware is mounted anywhere inside the plugin
-    loop the two swap places - the plugin's span becomes the root and the
-    request span its child - which is exactly the orphaning this is meant to
-    prevent, and which no attribute assertion notices.
-    """
-    ds.pm.register(_MiddlewarePlugin(), name="httpspan-middleware")
-    try:
-        otel_spans.clear()
-        response = await ds.client.get(f"/{ds.db_name}/t")
-        assert response.status_code == 200
-    finally:
-        ds.pm.unregister(name="httpspan-middleware")
-
-    spans = otel_spans.get_finished_spans()
-    server = [span for span in spans if span.kind is SpanKind.SERVER]
-    assert len(server) == 1, "expected exactly one SERVER span per request"
-    server_span = server[0]
-    assert server_span.parent is None, "the request span should be the trace root"
-
-    plugin_spans = [span for span in spans if span.name == PLUGIN_MIDDLEWARE_SPAN]
-    assert len(plugin_spans) == 1
-    assert plugin_spans[0].parent is not None
-    assert plugin_spans[0].parent.span_id == server_span.context.span_id
-    assert plugin_spans[0].context.trace_id == server_span.context.trace_id
-
-    # And the database work is in the same trace, not off on its own.
-    queries = [span for span in spans if span.name == "db.query"]
-    assert queries, "a table page should have issued at least one query"
-    for query in queries:
-        assert query.context.trace_id == server_span.context.trace_id
-
-
-@pytest.mark.asyncio
-async def test_unrecognised_method_is_clamped(ds, otel_spans):
-    """
-    Anyone can send `FROB / HTTP/1.1`. An unclamped method is an unbounded
-    dimension a client controls, so semantic conventions map anything off the
-    known list to `_OTHER`.
-
-    The span name is checked too, and it is the reason the router clamps the
-    method a second time when it renames the span: the middleware's clamping
-    protects the attribute, but the name is rebuilt from `request.method` in
-    `route_path`, which is the raw client string. An unclamped rename would
-    put attacker-supplied text straight back into the span name.
-    """
-    otel_spans.clear()
-    await ds.client.request("FROB", f"/{ds.db_name}/t")
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    assert server[0].attributes["http.request.method"] == "_OTHER"
-    assert server[0].name == f"_OTHER {server[0].attributes['http.route']}"
-
-
-@pytest.mark.asyncio
-async def test_known_method_is_not_clamped(ds, otel_spans):
-    "The other half of clamping: a real method must survive it verbatim."
-    otel_spans.clear()
-    await ds.client.get(f"/{ds.db_name}/t")
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    assert server[0].attributes["http.request.method"] == "GET"
-    assert server[0].name == f"GET {server[0].attributes['http.route']}"
-
-
-@pytest.mark.asyncio
-async def test_the_query_string_is_never_recorded(ds, otel_spans):
-    """
-    Datasette puts user-supplied SQL in `?sql=` and canned query parameters in
-    the query string, so no span may carry it. Asserting on the absence of a
-    `url.query` key alone would not catch it arriving under some other name,
-    so this searches every attribute value of every span for the marker.
-    """
-    marker = "canary-9f2b1c"
-    otel_spans.clear()
-    await ds.client.get(f"/{ds.db_name}/t?_facet=v&_nosuch={marker}")
-    spans = otel_spans.get_finished_spans()
-    assert _server_spans(otel_spans), "no request span was emitted"
-    leaked = [
-        f"{span.name} -> {key}={value!r}"
-        for span in spans
-        for key, value in (span.attributes or {}).items()
-        if marker in str(value) or key == "url.query"
-    ]
-    assert not leaked, "the query string reached a span attribute: " + ", ".join(leaked)
-
-
-@pytest.mark.asyncio
-async def test_url_path_is_recorded_without_the_query_string(ds, otel_spans):
-    otel_spans.clear()
-    await ds.client.get(f"/{ds.db_name}/t?_facet=v")
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    assert server[0].attributes["url.path"] == f"/{ds.db_name}/t"
-
-
-@pytest.mark.asyncio
-async def test_escaping_exception_sets_error_type_and_reraises(ds, otel_spans):
-    """
-    An exception that gets past `route_path` must be recorded, not swallowed.
-
-    No response ever started, so there is no status code to record either.
-    """
-    ds.pm.register(
-        _RaisingMiddlewarePlugin(call_app_first=False), name="httpspan-raiser"
-    )
-    try:
-        otel_spans.clear()
-        with pytest.raises(RuntimeError):
-            await ds.client.get(f"/{ds.db_name}/t")
-    finally:
-        ds.pm.unregister(name="httpspan-raiser")
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    assert server[0].attributes["error.type"] == "RuntimeError"
-    assert "http.response.status_code" not in server[0].attributes
-    assert server[0].status.status_code is StatusCode.ERROR
-
-
-@pytest.mark.asyncio
-async def test_an_escaping_exception_beats_the_status_code_for_error_type(
-    ds, otel_spans
-):
-    """
-    Both paths can fire on one request: a 500 response is sent and *then*
-    something raises on the way out. The `finally` block runs while the
-    exception is propagating, so without the guard it would overwrite the
-    exception's class name with the string "500" - strictly less information
-    about what actually went wrong.
-    """
-    ds.pm.register(_BoomPlugin(), name="httpspan-boom")
-    ds.pm.register(
-        _RaisingMiddlewarePlugin(call_app_first=True), name="httpspan-raiser"
-    )
-    try:
-        otel_spans.clear()
-        with pytest.raises(RuntimeError):
-            await ds.client.get("/-/http-span-boom")
-    finally:
-        ds.pm.unregister(name="httpspan-raiser")
-        ds.pm.unregister(name="httpspan-boom")
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    # The 500 really was sent, so the status is still recorded ...
-    assert server[0].attributes["http.response.status_code"] == 500
-    # ... but error.type names the exception, not the status.
-    assert server[0].attributes["error.type"] == "RuntimeError"
-
-
-@pytest.mark.asyncio
-async def test_a_404_is_not_an_error(ds, otel_spans):
-    """
-    Per semantic conventions a 4xx is the client's mistake, not the server's,
-    so a SERVER span must record the status and leave both its own status and
-    `error.type` alone. Datasette 404s are routine - every missing table, and
-    every bot probing for /wp-login.php - so treating them as errors would
-    drown a real 500 in noise.
-
-    Note this 404 *does* match a route: `/no-such-database-at-all` matches the
-    database pattern and the view then raises `NotFound`. Most Datasette 404s
-    are that shape rather than the unrouted one below.
-    """
-    otel_spans.clear()
-    response = await ds.client.get("/no-such-database-at-all")
-    assert response.status_code == 404
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    assert server[0].attributes["http.response.status_code"] == 404
-    assert "error.type" not in server[0].attributes
-    assert server[0].status.status_code is StatusCode.UNSET
-
-
-@pytest.mark.asyncio
-async def test_an_unrouted_404_has_no_route_and_a_bare_method_name(ds, otel_spans):
-    """
-    When no route matches there is nothing to set `http.route` to, so the span
-    keeps the bare method name it was given at the edge - which is exactly the
-    fallback semantic conventions specify for an unknown route.
-
-    `/a/b/c/d/e` is used rather than a plausible-looking missing name because
-    Datasette's route table is greedy: `/no-such-database-at-all` matches the
-    database pattern, and `/-/nope/deeper` matches the row pattern. Only a
-    path deeper than any route matches nothing at all.
-    """
-    otel_spans.clear()
-    response = await ds.client.get("/a/b/c/d/e")
-    assert response.status_code == 404
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    assert server[0].name == "GET"
-    assert "http.route" not in server[0].attributes
-    assert server[0].attributes["http.response.status_code"] == 404
-    assert server[0].status.status_code is StatusCode.UNSET
-
-
-@pytest.mark.asyncio
-async def test_only_the_first_http_response_start_is_recorded(otel_spans):
-    """
-    The `send` wrapper keeps the first status it sees.
-
-    Nothing in Datasette sends two `http.response.start` messages, so this
-    drives the middleware directly rather than pretending a request could
-    reach it. Without the guard a misbehaving plugin's second start message
-    would silently replace the status the client actually received.
-    """
-
-    async def two_starts(scope, receive, send):
-        await send({"type": "http.response.start", "status": 200, "headers": []})
-        await send({"type": "http.response.start", "status": 503, "headers": []})
-        await send({"type": "http.response.body", "body": b""})
-
-    middleware = TelemetryMiddleware(two_starts)
-    scope = {
-        "type": "http",
-        "method": "GET",
-        "path": "/twice",
-        "raw_path": b"/twice",
-        "scheme": "http",
-        "headers": [],
-    }
-    otel_spans.clear()
-    await middleware(scope, None, lambda message: asyncio.sleep(0))
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    assert server[0].attributes["http.response.status_code"] == 200
-    assert "error.type" not in server[0].attributes
-
-
-@pytest.mark.asyncio
-async def test_lifespan_scope_passes_through_unspanned(otel_spans):
-    """
-    `AsgiLifespan` sits *inside* this middleware, so the scope-type check has
-    to come first or startup and shutdown events never reach it. A SERVER
-    span for a lifespan scope is the symptom of that check being missing or
-    late.
-    """
-    instance = Datasette(memory=True)
-    app = instance.app()
-    events = iter([{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}])
-    sent = []
-
-    async def receive():
-        return next(events)
-
-    async def send(message):
-        sent.append(message["type"])
-
-    otel_spans.clear()
-    await app({"type": "lifespan"}, receive, send)
-    assert sent == ["lifespan.startup.complete", "lifespan.shutdown.complete"]
-    assert not _server_spans(otel_spans)
-
-
-@pytest.mark.asyncio
-async def test_http_route_is_the_compiled_pattern(ds, otel_spans):
-    """
-    `http.route` is the route's compiled regex, not a prettified template.
-
-    Asserted against what Datasette's own router resolves rather than against
-    a copied literal, so this pins the *relationship* - the attribute is the
-    matched route - and does not break when a core pattern is edited.
-    """
-    path = f"/{ds.db_name}/t"
-    expected = _route_for(ds, path)
-    otel_spans.clear()
-    assert (await ds.client.get(path)).status_code == 200
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    assert server[0].attributes["http.route"] == expected
-    assert server[0].name == f"GET {expected}"
-    # The pattern really is the ugly one, and that is deliberate - if someone
-    # adds a prettifier this is the assertion that should make them argue for
-    # it rather than slip it in.
-    assert "(?P<database>" in expected
-
-
-@pytest.mark.asyncio
-async def test_the_route_lands_on_the_request_span_not_a_plugins_current_span(
-    ds, otel_spans
-):
-    """
-    The route is set on the span the middleware started, found through the
-    ASGI scope - not on whatever span happens to be current when routing
-    resolves.
-
-    Those are the same span only until a plugin `asgi_wrapper()` starts one of
-    its own. A plugin wrapper runs *inside* this middleware, so an instrumented
-    plugin makes its span current for the whole request: reading the current
-    span in `route_path` renames that plugin's INTERNAL span to
-    `GET <route>` and hangs `http.route` off it, while the actual request span
-    keeps a bare method name and never gets the one attribute a trace UI
-    groups requests by. Verified by reproducing it, not by reasoning about it.
-    """
-    ds.pm.register(_MiddlewarePlugin(), name="httpspan-middleware")
-    try:
-        otel_spans.clear()
-        path = f"/{ds.db_name}/t"
-        expected = _route_for(ds, path)
-        assert (await ds.client.get(path)).status_code == 200
-    finally:
-        ds.pm.unregister(name="httpspan-middleware")
-
-    spans = otel_spans.get_finished_spans()
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    assert server[0].attributes["http.route"] == expected
-    assert server[0].name == f"GET {expected}"
-    # And the plugin's span is untouched: same name, no route attribute.
-    plugin_spans = [span for span in spans if span.name == PLUGIN_MIDDLEWARE_SPAN]
-    assert len(plugin_spans) == 1
-    assert "http.route" not in (plugin_spans[0].attributes or {})
-
-
-@pytest.mark.asyncio
-async def test_request_span_attributes(ds, otel_spans):
-    "The whole attribute set on one ordinary request."
-    path = f"/{ds.db_name}/t"
-    otel_spans.clear()
-    assert (await ds.client.get(path)).status_code == 200
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    attributes = server[0].attributes
-    assert attributes["http.request.method"] == "GET"
-    assert attributes["url.path"] == path
-    assert attributes["url.scheme"] == "http"
-    assert attributes["http.response.status_code"] == 200
-    assert attributes["http.route"] == _route_for(ds, path)
-    assert server[0].status.status_code is StatusCode.UNSET
-    # Never, on any span: an IP is borderline PII and the query string carries
-    # user-supplied SQL.
-    assert "client.address" not in attributes
-    assert "url.query" not in attributes
-
-
-@pytest.mark.asyncio
-async def test_db_query_spans_are_children_of_the_request_span(ds, otel_spans):
-    """
-    The point of the whole PR.
-
-    Not just "same trace ID" - every `db.query` span must reach the request
-    span by walking parents, and the request span must be the only root. A
-    stray root would show up in a trace UI as its own single-span trace, which
-    is the state this replaces.
-    """
-    otel_spans.clear()
-    assert (await ds.client.get(f"/{ds.db_name}/t?_facet=v")).status_code == 200
-    spans = otel_spans.get_finished_spans()
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    server_span = server[0]
-    assert server_span.parent is None
-
-    by_span_id = {span.context.span_id: span for span in spans}
-    roots = [span for span in spans if span.parent is None]
-    assert [span.name for span in roots] == [server_span.name], (
-        "every span from a request should hang off the request span, but these "
-        f"are roots: {sorted(span.name for span in roots)}"
-    )
-
-    queries = [span for span in spans if span.name == "db.query"]
-    assert queries, "a faceted table page should have issued queries"
-    for query in queries:
-        assert query.context.trace_id == server_span.context.trace_id
-        # Walk up to the root, which must be the request span.
-        current = query
-        seen = 0
-        while current.parent is not None:
-            current = by_span_id[current.parent.span_id]
-            seen += 1
-            assert seen < 20, "parent chain did not terminate"
-        assert current is server_span
-
-
-@pytest.mark.asyncio
-async def test_500_sets_error_status_and_error_type(ds, otel_spans):
-    """
-    A plain 500 - no exception escaping the app, because `route_path` converts
-    it into a response itself. The status is the only signal the middleware
-    gets, so `error.type` is the status as a string.
-    """
-    ds.pm.register(_BoomPlugin(), name="httpspan-boom")
-    try:
-        otel_spans.clear()
-        response = await ds.client.get("/-/http-span-boom")
-        assert response.status_code == 500
-    finally:
-        ds.pm.unregister(name="httpspan-boom")
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    assert server[0].attributes["http.response.status_code"] == 500
-    assert server[0].attributes["error.type"] == "500"
-    assert server[0].status.status_code is StatusCode.ERROR
-
-
-@pytest.mark.asyncio
-async def test_csv_stream_span_covers_the_body_send(ds_paging, otel_spans):
-    """
-    The span must not end when the handler returns - it has to cover the
-    response body.
-
-    `stream_csv` runs its generator inline inside `AsgiStream.asgi_send`, and
-    that call happens inside the single `await self.app(...)` the middleware
-    makes, so a plain `finally` is enough and no deferred-end machinery is
-    needed. This is the assertion that holds that claim up: a `db.query` that
-    starts during the body send must still finish before the request span
-    does.
-
-    Only meaningful on an export that actually pages, hence `ds_paging` - on a
-    single-page table every query is over before the body begins and this
-    passes however early the span ends. The middle assertion below, that some
-    query *started* after `http.response.start` went out, is what keeps the
-    test honest about that; it is why the app is driven as raw ASGI rather
-    than through `ds.client`, which cannot timestamp the response start.
-
-    `time.time_ns()` is the same clock the SDK stamps spans with, so the two
-    are directly comparable.
-    """
-    app = ds_paging.app()
-    body = []
-    response_started_at = None
-
-    async def receive():
-        return {"type": "http.request", "body": b"", "more_body": False}
-
-    async def send(message):
-        nonlocal response_started_at
-        if message["type"] == "http.response.start":
-            assert message["status"] == 200
-            response_started_at = time.time_ns()
-        else:
-            body.append(message.get("body") or b"")
-
-    otel_spans.clear()
-    await app(
-        {
-            "type": "http",
-            "http_version": "1.1",
-            "method": "GET",
-            "path": f"/{ds_paging.db_name}/t.csv",
-            "raw_path": f"/{ds_paging.db_name}/t.csv".encode("latin-1"),
-            "query_string": b"_stream=1",
-            "scheme": "http",
-            "headers": [(b"host", b"localhost")],
-        },
-        receive,
-        send,
-    )
-    # 40 rows plus a header - the export really did read past one page
-    assert len(b"".join(body).decode("utf-8").strip().splitlines()) == 41
-    assert response_started_at is not None
-
-    spans = otel_spans.get_finished_spans()
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    server_span = server[0]
-    queries = [span for span in spans if span.name == "db.query"]
-    assert len(queries) > 1
-    during_body = [span for span in queries if span.start_time > response_started_at]
-    assert during_body, (
-        "no query ran after the response started, so this workload cannot "
-        "distinguish a span that covers the body send from one that ends when "
-        "the handler returns - the export is not paging"
-    )
-    last_query_end = max(span.end_time for span in queries)
-    assert server_span.end_time > last_query_end, (
-        "the request span ended before the last query of a streaming export - "
-        "it is not covering the response body"
-    )
-    for query in queries:
-        assert query.context.trace_id == server_span.context.trace_id
-
-
-@pytest.mark.asyncio
-async def test_inbound_traceparent_becomes_the_parent(ds, otel_spans):
-    """
-    W3C trace context is extracted with the global propagator, so a request
-    from an already-traced caller continues that trace.
-
-    The sampled flag has to be set: the SDK's default sampler is
-    parentbased_always_on, so a `-00` flag would drop the span and the test
-    would fail for a reason that has nothing to do with propagation.
-    """
-    trace_id = "4bf92f3577b34da6a3ce929d0e0e4736"
-    parent_span_id = "00f067aa0ba902b7"
-    otel_spans.clear()
-    response = await ds.client.get(
-        f"/{ds.db_name}/t",
-        headers={"traceparent": f"00-{trace_id}-{parent_span_id}-01"},
-    )
-    assert response.status_code == 200
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    server_span = server[0]
-    assert f"{server_span.context.trace_id:032x}" == trace_id
-    assert server_span.parent is not None
-    assert f"{server_span.parent.span_id:016x}" == parent_span_id
-    assert server_span.parent.is_remote
-    # And the database spans joined the caller's trace too, not a new one.
-    queries = [
-        span for span in otel_spans.get_finished_spans() if span.name == "db.query"
-    ]
-    assert queries
-    for query in queries:
-        assert f"{query.context.trace_id:032x}" == trace_id
-
-
-@pytest.mark.asyncio
-async def test_user_supplied_sql_in_the_query_string_is_never_recorded(ds, otel_spans):
-    """
-    The `?sql=` case specifically, which is the one that matters: this is the
-    request where the query string *is* user-supplied SQL, and it reaches a
-    view that runs it. The marker is searched for across every attribute of
-    every span in the trace, not just for a `url.query` key, so recording it
-    under some other name fails too.
-
-    `db.query.text` legitimately contains the SQL - that is documented and
-    deliberate - so the marker is checked against the request span's own
-    attributes, and against `url.*` and `http.*` keys everywhere.
-    """
-    marker = "secret_marker_5b1f"
-    otel_spans.clear()
-    # `/{db}?sql=` 302s to the query view, so go straight there - a redirect
-    # would leave the SQL only on a span for a request that never ran it.
-    response = await ds.client.get(f"/{ds.db_name}/-/query?sql=select+'{marker}'")
-    assert response.status_code == 200
-    spans = otel_spans.get_finished_spans()
-    server = _server_spans(otel_spans)
-    assert len(server) == 1
-    leaked = [
-        f"{span.name} -> {key}={value!r}"
-        for span in spans
-        for key, value in (span.attributes or {}).items()
-        if (span is server[0] or str(key).startswith(("url.", "http.")))
-        and (marker in str(value) or str(key) == "url.query")
-    ]
-    assert not leaked, "the query string reached a span attribute: " + ", ".join(leaked)
-    # The request really did carry the marker, so the search above had
-    # something to find.
-    assert marker in response.text
-
-
-def test_request_span_skips_a_valid_but_non_recording_span():
-    """
-    `request_span()` is guarded on `is_recording()`, not on
-    `get_span_context().is_valid`, and this is the case that separates them.
-
-    With no provider installed but an inbound `traceparent`, the API's
-    NoOpTracer hands back a `NonRecordingSpan` carrying the *remote* span
-    context - valid, sampled, and recording nothing. An `is_valid` guard would
-    wave that through and the router would build the name string and call
-    `set_attribute`/`update_name` on a span that discards both.
-
-    Tested at this level deliberately: through a real request the two guards
-    are indistinguishable, because every call the router makes on a
-    NonRecordingSpan is already a no-op. The only difference is the work done
-    to get there, so the guard itself is what has to be asserted on.
-    """
-    remote = SpanContext(
-        trace_id=0x4BF92F3577B34DA6A3CE929D0E0E4736,
-        span_id=0x00F067AA0BA902B7,
-        is_remote=True,
-        trace_flags=TraceFlags(TraceFlags.SAMPLED),
-    )
-    assert remote.is_valid
-    non_recording = NonRecordingSpan(remote)
-    assert non_recording.is_recording() is False
-    assert request_span({REQUEST_SPAN_SCOPE_KEY: non_recording}) is None
-    # Nothing current, nothing in the scope: the INVALID_SPAN fallback.
-    assert request_span({}) is None
-    # And the case it must not skip.
-    with tracer.start_as_current_span("test.request_span.recording") as span:
-        assert request_span({REQUEST_SPAN_SCOPE_KEY: span}) is span
-        # Falling back to the current span is how an externally installed
-        # SERVER span still gets enriched.
-        assert request_span({}) is span
-
-
-NO_PROVIDER_PROGRAM = textwrap.dedent("""
-    import asyncio, json, sys
-
-    from datasette.telemetry import TelemetryMiddleware
-
-    seen = {}
-
-
-    async def inner(scope, receive, send):
-        seen.setdefault("sends", []).append(send)
-        seen.setdefault("scopes", []).append(scope)
-        await send({"type": "http.response.start", "status": 200, "headers": []})
-        await send({"type": "http.response.body", "body": b""})
-
-
-    async def real_send(message):
-        pass
-
-
-    async def main():
-        middleware = TelemetryMiddleware(inner)
-        for headers in ([], [(b"traceparent", b"00-" + b"a" * 32 + b"-" + b"b" * 16 + b"-01")]):
-            await middleware(
-                {
-                    "type": "http",
-                    "method": "GET",
-                    "path": "/",
-                    "raw_path": b"/",
-                    "scheme": "http",
-                    "headers": headers,
-                },
-                None,
-                real_send,
-            )
-        print(
-            json.dumps(
-                {
-                    "unwrapped": [send is real_send for send in seen["sends"]],
-                    "scope_keys": [
-                        "datasette.telemetry.request_span" in scope
-                        for scope in seen["scopes"]
-                    ],
-                    "sdk_imported": any(
-                        name.startswith("opentelemetry.sdk") for name in sys.modules
-                    ),
-                }
-            )
-        )
-
-
-    asyncio.run(main())
-    """)
-
-
-def test_no_provider_takes_the_fast_path():
-    """
-    With no `TracerProvider` installed the middleware must hand the
-    application the *original* `send`, not a wrapper - a default Datasette
-    install should pay essentially nothing for instrumentation it is not
-    using.
-
-    This has to run in a subprocess. The suite's `_otel_provider` fixture is
-    session-scoped and autouse, and `set_tracer_provider()` is effectively
-    once-per-process, so in-process every span is recording and the fast path
-    is unreachable.
-
-    The second case, with an inbound `traceparent`, is the one that pins the
-    check itself. With no provider the API's NoOpTracer returns a
-    NonRecordingSpan carrying the *remote* span context: its
-    `get_span_context().is_valid` is True while `is_recording()` is False. A
-    fast path guarded on `is_valid` would therefore silently stop working for
-    exactly the requests that arrive from an already-traced caller - which on
-    a real deployment behind an instrumented proxy is all of them.
-
-    conftest.py's pytest_collection_modifyitems() moves this test to the front
-    of the run by name - if you rename it, rename it there too.
-    """
-    result = subprocess.run(
-        [sys.executable, "-c", NO_PROVIDER_PROGRAM],
-        capture_output=True,
-        text=True,
-        check=True,
-    )
-    report = json.loads(result.stdout)
-    assert report["sdk_imported"] is False, "the SDK loaded in a fresh interpreter"
-    assert report["unwrapped"] == [True, True], (
-        "the middleware wrapped `send` with no provider installed; the second "
-        "entry is the inbound-traceparent case, which fails if the fast path "
-        "is guarded on is_valid instead of is_recording()"
-    )
-    # Same fast path, other observable: nothing is stashed in the scope either.
-    assert report["scope_keys"] == [False, False]

tests/test_internals_database.py

@@ -3,13 +3,11 @@ Tests for the datasette.database.Database class
 """
 
 import asyncio
-import threading
 import uuid
 from types import SimpleNamespace
 
 import pytest
 import sqlite_utils
-from opentelemetry import context as otel_context_api
 
 from datasette.app import Datasette
 from datasette.database import (
@@ -17,11 +15,16 @@ from datasette.database import (
     DatasetteClosedError,
     ExecuteWriteResult,
     MultipleValues,
+    QueryInterrupted,
     Results,
     _deliver_write_result,
 )
 from datasette.utils import Column
-from datasette.utils.sqlite import sqlite3, supports_returning
+from datasette.utils.sqlite import (
+    sqlite3,
+    sqlite_derived_table_dependencies,
+    supports_returning,
+)
 
 requires_sqlite_returning = pytest.mark.skipif(
     not supports_returning(), reason="SQLite does not support RETURNING"
@@ -40,6 +43,31 @@ async def test_execute(db):
     assert 15 == len(results)
 
 
+@pytest.mark.asyncio
+async def test_derived_dependency_cache_survives_failed_refresh(monkeypatch):
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.derived_table_dependencies()
+    previous_cache = db._cached_derived_table_dependencies
+    await db.execute_write("create table dependency_cache_refresh (id integer)")
+
+    class UnavailableSchema:
+        def execute(self, sql):
+            raise sqlite3.DatabaseError("schema temporarily unavailable")
+
+    with monkeypatch.context() as patch:
+        patch.setattr(
+            "datasette.database.sqlite_derived_table_dependencies",
+            lambda conn: sqlite_derived_table_dependencies(UnavailableSchema()),
+        )
+        with pytest.raises(sqlite3.DatabaseError, match="schema temporarily"):
+            await db.derived_table_dependencies()
+    assert db._cached_derived_table_dependencies == previous_cache
+
+    await db.derived_table_dependencies()
+    assert db._cached_derived_table_dependencies[0] != previous_cache[0]
+
+
 @pytest.mark.asyncio
 async def test_results_first(db):
     assert None is (await db.execute("select * from facetable where pk > 100")).first()
@@ -480,6 +508,31 @@ async def test_view_names(db):
     ]
 
 
+@pytest.mark.asyncio
+async def test_execute_write_custom_time_limit():
+    ds = Datasette(settings={"sql_time_limit_ms": 1})
+    db = ds.add_memory_database(uuid.uuid4().hex, name="write_limits")
+    await ds.invoke_startup()
+    # Bounded work from PR #51; even without a limit this finishes on its own.
+    sql = (
+        "with recursive c(x) as "
+        "(select 1 union all select x+1 from c where x < 800000) "
+        "select x from c where x < 0"
+    )
+    try:
+        await db.execute_write("create table items(value integer)")
+        with pytest.raises(QueryInterrupted):
+            await db.execute(sql)
+        # Writes take their own explicit limit, independent of the read setting.
+        with pytest.raises(QueryInterrupted):
+            await db.execute_write(f"insert into items(value) {sql}", time_limit_ms=1)
+        # Interruption must leave the connection available for subsequent writes.
+        await db.execute_write("insert into items(value) values (1)")
+        assert (await db.execute("select value from items")).single_value() == 1
+    finally:
+        ds.close()
+
+
 @pytest.mark.asyncio
 async def test_execute_write_block_true(db):
     result = await db.execute_write(
@@ -707,6 +760,33 @@ async def test_execute_write_fn_block_false(db):
     assert isinstance(task_id, uuid.UUID)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("disable_threads", (False, True))
+async def test_execute_write_fn_block_false_returns_uuid(tmp_path, disable_threads):
+    # block=False is documented to return "a UUID representing the queued task".
+    # With num_sql_threads=0 there is no write thread, so the non-threaded branch
+    # has to satisfy the same contract as the threaded one.
+    settings = {"num_sql_threads": 0} if disable_threads else {}
+    ds = Datasette([], memory=True, settings=settings)
+    await ds.invoke_startup()
+    db = ds.add_memory_database("test_block_false")
+    await db.execute_write(
+        "create table if not exists t (id integer primary key, v text)"
+    )
+
+    def write_fn(conn):
+        conn.execute("insert into t (v) values ('a')")
+        # Returns None, like most write functions.
+
+    task_id = await db.execute_write_fn(write_fn, block=False)
+
+    assert isinstance(task_id, uuid.UUID)
+    # Distinct per call, so a caller can tell two queued tasks apart.
+    second = await db.execute_write_fn(write_fn, block=False)
+    assert isinstance(second, uuid.UUID)
+    assert second != task_id
+
+
 @pytest.mark.asyncio
 async def test_execute_write_fn_block_true(db):
     def write_fn(conn):
@@ -1225,110 +1305,3 @@ async def test_database_close_is_idempotent(tmpdir):
     # Second call should be a no-op, not raise
     db.close()
     ds._internal_database.close()
-
-
-_CONTEXT_LEAK_MARKER_KEY = "otel-context-leak-marker"
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize("num_sql_threads", (0, 1))
-async def test_write_thread_context_is_detached_between_tasks(
-    tmp_path, monkeypatch, num_sql_threads
-):
-    """
-    The write thread attaches each task's otel Context and must detach it
-    again before picking up the next task. The thread is persistent and
-    shared, so a leaked token would grow that thread's context stack for the
-    rest of the process - and a *wrong*-token detach only logs a warning
-    rather than raising, so "does it throw" cannot catch either mistake.
-
-    Two things are asserted, because neither alone is sufficient:
-
-    1. Each task observes the context value that was current on the event
-       loop when it was queued. This is what fails if the Context is not
-       carried on WriteTask, or is never attached. It does *not* catch a
-       missing detach: attach() replaces the current Context wholesale, so a
-       leftover one from a previous task is simply overwritten.
-    2. The write thread's attach depth is identical at the same point in
-       every task. This is what fails if detach is missing - the stack grows
-       by one per task - and it holds across a task that raises, because the
-       detach lives in a `finally`.
-
-    An otel context value is used rather than a plain contextvars.ContextVar:
-    a plain var set on the event loop never crosses into the write thread, so
-    the probe would read None every time and the test could not fail.
-    """
-    name = f"context_leak_test_{num_sql_threads}"
-    db_path = tmp_path / f"{name}.db"
-    sqlite3.connect(db_path).close()
-    ds = Datasette([str(db_path)], settings={"num_sql_threads": num_sql_threads})
-    db = ds.get_database(name)
-    await db.execute_write("create table t (id integer primary key)")
-
-    write_thread_name = f"_execute_writes for database {name}"
-    depth = {"value": 0}
-    real_attach = otel_context_api.attach
-    real_detach = otel_context_api.detach
-
-    def counting_attach(context):
-        token = real_attach(context)
-        if threading.current_thread().name == write_thread_name:
-            depth["value"] += 1
-        return token
-
-    def counting_detach(token):
-        real_detach(token)
-        if threading.current_thread().name == write_thread_name:
-            depth["value"] -= 1
-
-    # Patched on the opentelemetry.context module itself, which is what both
-    # database.py and opentelemetry.trace.use_span() look the functions up on.
-    monkeypatch.setattr(otel_context_api, "attach", counting_attach)
-    monkeypatch.setattr(otel_context_api, "detach", counting_detach)
-
-    seen_markers = []
-    seen_depths = []
-
-    def probe(conn):
-        seen_markers.append(otel_context_api.get_value(_CONTEXT_LEAK_MARKER_KEY))
-        seen_depths.append(depth["value"])
-
-    def failing_probe(conn):
-        probe(conn)
-        # Exercises the write thread's exception path: the detach still has
-        # to happen, which is why it lives in a `finally`.
-        raise ValueError("deliberate failure inside a write task")
-
-    try:
-        for i in range(5):
-            ctx = otel_context_api.set_value(_CONTEXT_LEAK_MARKER_KEY, f"marker-{i}")
-            token = real_attach(ctx)
-            try:
-                if i == 2:
-                    with pytest.raises(ValueError):
-                        await db.execute_write_fn(failing_probe)
-                else:
-                    await db.execute_write_fn(probe)
-            finally:
-                real_detach(token)
-
-        # Sanity check: no marker is active in *this* (event loop) context
-        # right now, so the final probe is a fair test of the write thread's
-        # own state rather than something this test forgot to clean up.
-        assert otel_context_api.get_value(_CONTEXT_LEAK_MARKER_KEY) is None
-        await db.execute_write_fn(probe)
-    finally:
-        db.close()
-
-    assert seen_markers == [
-        "marker-0",
-        "marker-1",
-        "marker-2",
-        "marker-3",
-        "marker-4",
-        None,
-    ]
-    assert len(set(seen_depths)) == 1, (
-        f"write thread context stack grew across tasks: {seen_depths} - "
-        "a token was attached without being detached"
-    )

tests/test_internals_datasette_client.py

@@ -1,4 +1,4 @@
-import httpx
+import httpx2
 import pytest
 import pytest_asyncio
 
@@ -43,7 +43,7 @@ async def datasette_with_permissions():
 async def test_client_methods(datasette, method, path, expected_status):
     client_method = getattr(datasette.client, method)
     response = await client_method(path)
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
     assert response.status_code == expected_status
     # Try that again using datasette.client.request
     response2 = await datasette.client.request(method, path)
@@ -63,7 +63,7 @@ async def test_client_post(datasette, prefix):
                 "message": "A message",
             },
         )
-        assert isinstance(response, httpx.Response)
+        assert isinstance(response, httpx2.Response)
         assert response.status_code == 302
         assert "ds_messages" in response.cookies
     finally:
@@ -135,7 +135,7 @@ async def test_skip_permission_checks_all_methods(datasette_with_permissions, me
     response = await client_method("/test_db.json", skip_permission_checks=True)
     # We don't check status code since some methods might not be allowed,
     # but we verify the request doesn't fail due to permissions
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
 
 
 @pytest.mark.asyncio
@@ -340,7 +340,7 @@ async def test_actor_parameter_all_http_methods(datasette, method):
     client_method = getattr(datasette.client, method)
     # Just verify no TypeError about unexpected 'actor' kwarg
     response = await client_method("/", actor={"id": "root"})
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
 
 
 @pytest.mark.asyncio

tests/test_lifespan.py

@@ -5,7 +5,7 @@ into the ASGI lifespan protocol.
 These exercise Datasette._startup_sequence() via three different callers:
 - AsgiLifespan, by hand-driving lifespan.startup messages (no HTTP request)
 - AsgiRunOnFirstRequest, the fallback for hosts that never send lifespan
-  events (this is what DatasetteClient / plain httpx.ASGITransport uses)
+  events (this is what DatasetteClient / plain httpx2.ASGITransport uses)
 - Both at once, to prove startup hooks run at most once
 """
 
@@ -13,7 +13,7 @@ import asyncio
 import contextlib
 import sqlite3
 
-import httpx
+import httpx2
 import pytest
 
 from datasette import hookimpl
@@ -131,8 +131,8 @@ async def test_startup_runs_exactly_once_across_lifespan_and_first_request():
 
         # A first HTTP request (as if the host never sent lifespan events,
         # or lifespan already ran) should not run the hook again.
-        transport = httpx.ASGITransport(app=app)
-        async with httpx.AsyncClient(
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
             transport=transport, base_url="http://localhost"
         ) as client:
             response1 = await client.get("/-/versions.json")
@@ -149,13 +149,13 @@ async def test_startup_runs_exactly_once_across_lifespan_and_first_request():
 @pytest.mark.asyncio
 async def test_no_lifespan_first_request_still_triggers_startup():
     # Pin today's behavior: a client that never drives ASGI lifespan events
-    # at all (like httpx.ASGITransport, which DatasetteClient uses) still
+    # at all (like httpx2.ASGITransport, which DatasetteClient uses) still
     # gets startup armed by the AsgiRunOnFirstRequest fallback.
     ds = Datasette(memory=True)
     assert ds._startup_invoked is False
     app = ds.app()
-    transport = httpx.ASGITransport(app=app)
-    async with httpx.AsyncClient(
+    transport = httpx2.ASGITransport(app=app)
+    async with httpx2.AsyncClient(
         transport=transport, base_url="http://localhost"
     ) as client:
         response = await client.get("/-/versions.json")
@@ -197,8 +197,8 @@ async def test_concurrent_first_requests_all_wait_for_slow_startup():
     pm.register(SlowStartupPlugin(), name="slow_startup_plugin")
     try:
         app = ds.app()
-        transport = httpx.ASGITransport(app=app)
-        async with httpx.AsyncClient(
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
             transport=transport, base_url="http://localhost"
         ) as client:
             responses = await asyncio.gather(

tests/test_load_extensions.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from unittest import mock
 
 import pytest
 
@@ -20,6 +21,29 @@ def has_compiled_ext():
     return False
 
 
+@pytest.mark.parametrize("load_fails", (False, True))
+def test_load_extension_is_disabled(load_fails):
+    ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
+    connection = mock.Mock()
+    if load_fails:
+        connection.load_extension.side_effect = RuntimeError
+
+    if load_fails:
+        with pytest.raises(RuntimeError):
+            ds._prepare_connection(connection, "data")
+    else:
+        ds._prepare_connection(connection, "data")
+
+    # Extensions are loaded using the Python API, never via SQL
+    assert connection.load_extension.mock_calls == [
+        mock.call(COMPILED_EXTENSION_PATH),
+    ]
+    assert connection.enable_load_extension.mock_calls == [
+        mock.call(True),
+        mock.call(False),
+    ]
+
+
 @pytest.mark.asyncio
 @pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
 async def test_load_extension_default_entrypoint():
@@ -64,3 +88,20 @@ async def test_load_extension_multiple_entrypoints():
     response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+c()")
     assert response.status_code == 200
     assert response.json()["rows"][0][0] == "c"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
+async def test_sql_cannot_load_additional_extension():
+    ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
+
+    response = await ds.client.get(
+        "/_memory/-/query.json",
+        params={
+            "sql": "select load_extension(:path, :entrypoint)",
+            "path": COMPILED_EXTENSION_PATH,
+            "entrypoint": "sqlite3_ext_b_init",
+        },
+    )
+    assert response.status_code == 400
+    assert response.json()["error"] == "not authorized"

tests/test_permission_endpoints.py

@@ -494,3 +494,31 @@ async def test_execute_sql_requires_view_database():
         )
     finally:
         ds.pm.unregister(plugin)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("path", ["/-/allowed", "/-/allowed.json?action=view-table"])
+async def test_allowed_requires_view_instance(path):
+    """
+    GHSA-hp2x-vx2r-6vxg: /-/allowed should be gated like its /-/rules sibling.
+
+    An actor who is denied view-instance gets 403 from / and /-/rules, but
+    /-/allowed (HTML and JSON) currently returns 200 to the same actor.
+    """
+    ds = Datasette(config={"allow": {"id": "alice"}})
+    await ds.invoke_startup()
+    db = ds.add_memory_database("live")
+    await db.execute_write("CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY)")
+    await ds.refresh_schemas()
+
+    assert (await ds.client.get("/")).status_code == 403
+    assert (await ds.client.get("/-/rules.json?action=view-table")).status_code == 403
+
+    response = await ds.client.get(path)
+    assert response.status_code == 403
+
+    # Alice is still allowed
+    response = await ds.client.get(
+        path, cookies={"ds_actor": ds.client.actor_cookie({"id": "alice"})}
+    )
+    assert response.status_code == 200

tests/test_playwright.py

@@ -5,7 +5,7 @@ import subprocess
 import sys
 import time
 
-import httpx
+import httpx2
 import pytest
 
 from datasette.fixtures import write_fixture_database
@@ -34,11 +34,11 @@ def wait_for_server(process, url, timeout=30):
                 f"stderr:\n{stderr}"
             )
         try:
-            response = httpx.get(url, timeout=1.0)
+            response = httpx2.get(url, timeout=1.0)
             if response.status_code < 500:
                 return
             last_error = f"HTTP {response.status_code}: {response.text[:200]}"
-        except httpx.HTTPError as ex:
+        except httpx2.HTTPError as ex:
             last_error = repr(ex)
         time.sleep(0.1)
     if process.poll() is None:
@@ -336,7 +336,7 @@ def project_rows(datasette_server, **filters):
         "_shape": "objects",
         **{key: str(value) for key, value in filters.items()},
     }
-    response = httpx.get(f"{datasette_server}data/projects.json", params=params)
+    response = httpx2.get(f"{datasette_server}data/projects.json", params=params)
     response.raise_for_status()
     return response.json()["rows"]
 
@@ -348,7 +348,7 @@ def project_row(datasette_server, pk):
 
 
 def binary_file_blob(datasette_server, pk):
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/binary_files/{pk}.blob",
         params={"_blob_column": "data"},
     )
@@ -369,7 +369,7 @@ def bulk_default_rows(datasette_server, **filters):
         "_shape": "objects",
         **{key: str(value) for key, value in filters.items()},
     }
-    response = httpx.get(f"{datasette_server}data/bulk_defaults.json", params=params)
+    response = httpx2.get(f"{datasette_server}data/bulk_defaults.json", params=params)
     response.raise_for_status()
     return response.json()["rows"]
 
@@ -379,7 +379,7 @@ def upsert_item_rows(datasette_server, **filters):
         "_shape": "objects",
         **{key: str(value) for key, value in filters.items()},
     }
-    response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params)
+    response = httpx2.get(f"{datasette_server}data/upsert_items.json", params=params)
     response.raise_for_status()
     return response.json()["rows"]
 
@@ -473,7 +473,7 @@ def test_create_table_flow(page, datasette_server):
     page.wait_for_url("**/data/playwright_created")
     assert "playwright_created" in page.locator("h1").inner_text()
 
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/playwright_created.json?_extra=columns,column_types"
     )
     response.raise_for_status()
@@ -487,7 +487,7 @@ def test_create_table_flow(page, datasette_server):
     assert data["column_types"] == {
         "metadata": {"type": "json", "config": None},
     }
-    schema_response = httpx.get(
+    schema_response = httpx2.get(
         f"{datasette_server}data/-/query.json",
         params={
             "sql": (
@@ -603,7 +603,7 @@ def test_create_table_from_data_flow(page, datasette_server):
     dialog.locator(".table-create-save").click()
     page.wait_for_url("**/data/playwright_from_data")
 
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/playwright_from_data.json?_shape=objects"
     )
     response.raise_for_status()
@@ -639,7 +639,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
     dialog.locator(".table-create-save").click()
     page.wait_for_url("**/data/playwright_numeric_blanks")
 
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects"
     )
     response.raise_for_status()
@@ -648,7 +648,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
         {"name": "B", "score": None},
     ]
 
-    schema_response = httpx.get(
+    schema_response = httpx2.get(
         f"{datasette_server}data/-/query.json",
         params={
             "sql": (
@@ -856,7 +856,7 @@ def test_alter_table_flow(page, datasette_server):
 
     columns = []
     for _ in range(20):
-        response = httpx.get(f"{datasette_server}data/projects.json?_extra=columns")
+        response = httpx2.get(f"{datasette_server}data/projects.json?_extra=columns")
         response.raise_for_status()
         columns = response.json()["columns"]
         if "status" in columns:

tests/test_pr76_fts_policy.py

@@ -0,0 +1,140 @@
+"""Policy and compatibility coverage for PR #76, run against the fixed checkout."""
+
+import uuid
+
+import pytest
+
+from datasette.app import Datasette
+from datasette.resources import TableResource
+from datasette.utils.sqlite import sqlite3, sqlite_derived_table_dependencies
+
+
+@pytest.mark.parametrize("vocab_name", ["words", "name USING fts4aux", 'quoted"name'])
+@pytest.mark.parametrize(
+    "module,arguments",
+    [
+        ("fts5vocab", "'Search,Index', 'row'"),
+        ("fts5vocab", "'SEARCH,INDEX', 'col'"),
+        ("fts5vocab", "'Search,Index', 'instance'"),
+        ("fts4aux", "'Search,Index'"),
+    ],
+)
+def test_vocabulary_dependency_identity(module, arguments, vocab_name):
+    conn = sqlite3.connect(":memory:")
+    try:
+        fts = "fts5" if module == "fts5vocab" else "fts4"
+        conn.execute(f'create virtual table "Search,Index" using {fts}(body)')
+        quoted_name = '"' + vocab_name.replace('"', '""') + '"'
+        conn.execute(
+            f"create virtual table {quoted_name} USING /* module */ {module}({arguments})"
+        )
+        assert sqlite_derived_table_dependencies(conn)[vocab_name] == "Search,Index"
+    finally:
+        conn.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("module", ["fts5", "fts4"])
+@pytest.mark.parametrize("external_content", [False, True], ids=["one-hop", "two-hop"])
+@pytest.mark.parametrize(
+    "source_allowed,vocab_allowed", [(False, True), (True, False), (True, True)]
+)
+async def test_vocabulary_immediate_source_permissions(
+    module, external_content, source_allowed, vocab_allowed
+):
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "search": {
+                            "permissions": {
+                                "view-table": (
+                                    {"id": "reader"} if source_allowed else False
+                                )
+                            }
+                        },
+                        "words": {"permissions": {"view-table": vocab_allowed}},
+                    }
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.execute_write("create table documents(body text)")
+    options = "body, content='documents'" if external_content else "body"
+    await db.execute_write(f"create virtual table search using {module}({options})")
+    definition = (
+        "fts5vocab('SEARCH', 'row')" if module == "fts5" else "fts4aux('SEARCH')"
+    )
+    await db.execute_write(f"create virtual table words using {definition}")
+    await ds.invoke_startup()
+    try:
+        actor = {"id": "reader"}
+        expected = source_allowed and vocab_allowed and not external_content
+        for name in ("words", "WORDS"):
+            assert (
+                await ds.allowed(
+                    action="view-table",
+                    resource=TableResource("data", name),
+                    actor=actor,
+                )
+                is expected
+            )
+        resources = await ds.allowed_resources(
+            "view-table", parent="data", actor=actor, include_is_private=True
+        )
+        words = [r for r in resources.resources if r.child == "words"]
+        assert bool(words) is expected
+        if expected:
+            assert words[0].private
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "words")
+        )
+        # Dropping the source invalidates dependency metadata and remains denied.
+        await db.execute_write("drop table search")
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "words"), actor=actor
+        )
+    finally:
+        ds.close()
+
+
+@pytest.mark.parametrize(
+    "module,definition",
+    [
+        ("fts5", "fts5vocab('main', 'search', 'row')"),
+        ("fts4", "fts4aux('main', 'search')"),
+    ],
+)
+def test_cross_schema_vocabulary_is_unresolved(module, definition):
+    conn = sqlite3.connect(":memory:")
+    try:
+        conn.execute(f"create virtual table search using {module}(body)")
+        conn.execute(f"create virtual table temp.words using {definition}")
+        # Cross-schema ownership is not representable by the current map.
+        # The source is itself derived, so the immediate-source policy denies it.
+        assert (
+            sqlite_derived_table_dependencies(conn, schema="temp")["words"] == "words"
+        )
+    finally:
+        conn.close()
+
+
+@pytest.mark.parametrize(
+    "definition",
+    [
+        """CREATE VIRTUAL TABLE"words"USING"fts5vocab"('search', 'row')""",
+        """CREATE VIRTUAL TABLE[words]USING[fts5vocab]('search', 'row')""",
+        """CREATE VIRTUAL TABLE`words`USING`fts5vocab`('search', 'row')""",
+    ],
+)
+def test_vocabulary_quoted_token_boundaries(definition):
+    conn = sqlite3.connect(":memory:")
+    try:
+        conn.execute("create virtual table search using fts5(body)")
+        conn.execute(definition)
+        assert sqlite_derived_table_dependencies(conn)["words"] == "search"
+    finally:
+        conn.close()

tests/test_pr76_statistics_policy.py

@@ -0,0 +1,113 @@
+"""Statistics access policy and plugin replacement coverage for PR #76."""
+
+import uuid
+
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.permissions import PermissionSQL
+from datasette.resources import TableResource
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("scope", [None, "global", "database", "table", "root"])
+async def test_statistics_denied_despite_allow_rules(scope):
+    config = {"databases": {"data": {"tables": {"sqlite_stat1": {}}}}}
+    grant = {"view-table": True}
+    if scope == "global":
+        config["permissions"] = grant
+    elif scope == "database":
+        config["databases"]["data"]["permissions"] = grant
+    elif scope == "table":
+        config["databases"]["data"]["tables"]["sqlite_stat1"]["permissions"] = grant
+    ds = Datasette(memory=True, config=config)
+    ds.root_enabled = scope == "root"
+    actor = {"id": "root"} if scope == "root" else {"id": "reader"}
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.execute_write("create table items(value text)")
+    await db.execute_write("create index items_value on items(value)")
+    await db.execute_write("insert into items values ('example')")
+    await db.execute_write("analyze")
+    await ds.invoke_startup()
+    try:
+        assert "view-sqlite-statistics" not in ds.actions
+        for name in ("sqlite_stat1", "SQLITE_STAT1"):
+            assert not await ds.allowed(
+                action="view-table", resource=TableResource("data", name), actor=actor
+            )
+        for suffix in ("", ".json", ".csv"):
+            assert (
+                await ds.client.get(f"/data/sqlite_stat1{suffix}", actor=actor)
+            ).status_code == 403
+        resources = await ds.allowed_resources("view-table", parent="data", actor=actor)
+        assert "sqlite_stat1" not in {r.child for r in resources.resources}
+        assert "items" in {r.child for r in resources.resources}
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "table", ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"]
+)
+@pytest.mark.parametrize("default_deny", [False, True])
+async def test_statistics_names_denied(table, default_deny):
+    ds = Datasette(memory=True, default_deny=default_deny)
+    ds.root_enabled = True
+    await ds.invoke_startup()
+    try:
+        for name in (table, table.upper()):
+            assert not await ds.allowed(
+                action="view-table",
+                resource=TableResource("_memory", name),
+                actor={"id": "root"},
+            )
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_plugin_can_replace_statistics_policy():
+    class ReplacementPolicy:
+        @hookimpl
+        def permission_resources_sql(self, action, actor):
+            if action == "view-table":
+                return PermissionSQL(
+                    sql="SELECT 'data' AS parent, 'sqlite_stat1' AS child, :statistics_allowed AS allow, 'custom statistics policy' AS reason",
+                    params={"statistics_allowed": int(actor == {"id": "reader"})},
+                )
+
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.execute_write("create table items(value text)")
+    await db.execute_write("analyze")
+    await ds.invoke_startup()
+    name = "datasette.default_permissions.sqlite_statistics"
+    original = ds.pm.unregister(name=name)
+    assert original is not None
+    replacement = ReplacementPolicy()
+    ds.pm.register(replacement, name="test-replacement-statistics-policy")
+    try:
+        actor = {"id": "reader"}
+        assert await ds.allowed(
+            action="view-table",
+            resource=TableResource("data", "sqlite_stat1"),
+            actor=actor,
+        )
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "sqlite_stat1")
+        )
+        resources = await ds.allowed_resources(
+            "view-table", parent="data", actor=actor, include_is_private=True
+        )
+        stats = [r for r in resources.resources if r.child == "sqlite_stat1"]
+        assert len(stats) == 1 and stats[0].private
+        assert (
+            await ds.client.get("/data/sqlite_stat1.json", actor=actor)
+        ).status_code == 200
+        assert (await ds.client.get("/data/sqlite_stat1.json")).status_code == 403
+    finally:
+        ds.pm.unregister(replacement)
+        ds.pm.register(original, name=name)
+        ds.close()

tests/test_queries.py

@@ -3248,74 +3248,6 @@ async def test_execute_write_create_table_uses_create_table_permission():
     assert not await db.table_exists("should_not_exist")
 
 
-@pytest.mark.asyncio
-async def test_execute_write_create_view_uses_create_view_permission():
-    ds = Datasette(
-        memory=True,
-        default_deny=True,
-        config={
-            "permissions": {
-                "insert-row": {"id": "row-writer"},
-                "update-row": {"id": "row-writer"},
-            },
-            "databases": {
-                "data": {
-                    "permissions": {
-                        "view-database": {"id": ["creator", "row-writer"]},
-                        "execute-write-sql": {"id": ["creator", "row-writer"]},
-                        "create-view": {"id": "creator"},
-                    }
-                }
-            },
-        },
-    )
-    db = ds.add_memory_database("execute_write_create_view", name="data")
-    await db.execute_write("create table dogs (id integer primary key, name text)")
-    await ds.invoke_startup()
-
-    analysis_response = await ds.client.get(
-        "/data/-/execute-write/analyze",
-        actor={"id": "creator"},
-        params={"sql": "create view dog_names as select id, name from dogs"},
-    )
-    allowed_response = await ds.client.post(
-        "/data/-/execute-write",
-        actor={"id": "creator"},
-        json={"sql": "create view dog_names as select id, name from dogs"},
-    )
-    row_permission_response = await ds.client.post(
-        "/data/-/execute-write",
-        actor={"id": "row-writer"},
-        json={"sql": "create view should_not_exist as select id from dogs"},
-    )
-
-    assert analysis_response.status_code == 200
-    analysis_data = analysis_response.json()
-    assert analysis_data["ok"] is True
-    assert analysis_data["execute_disabled"] is False
-    assert analysis_data["analysis_rows"] == [
-        {
-            "operation": "create",
-            "database": "data",
-            "table": "dog_names",
-            "required_permission": "create-view",
-            "source": None,
-            "allowed": True,
-        }
-    ]
-
-    assert allowed_response.status_code == 200
-    assert allowed_response.json()["ok"] is True
-    assert allowed_response.json()["message"] == "Query executed"
-    assert await db.view_exists("dog_names")
-
-    assert row_permission_response.status_code == 403
-    assert row_permission_response.json()["errors"] == [
-        "Permission denied: need create-view on data"
-    ]
-    assert not await db.view_exists("should_not_exist")
-
-
 @pytest.mark.parametrize(
     (
         "database_name",

tests/test_schema_endpoints.py

@@ -246,3 +246,114 @@ async def test_table_not_exists(schema_ds):
     response = await schema_ds.client.get("/schema_public_db/nonexistent/-/schema.md")
     assert response.status_code == 404
     assert "not found" in response.text.lower()
+
+
+@pytest_asyncio.fixture(scope="module")
+async def schema_table_perms_ds():
+    """
+    A database that is viewable by anonymous users, but with one table
+    locked down using the documented per-table lockdown recipe:
+    a table-level allow block combined with allow_sql: false.
+    """
+    ds = Datasette(
+        config={
+            "databases": {
+                "schema_table_perms_db": {
+                    "allow_sql": False,
+                    "tables": {"employee_salaries": {"allow": {"id": "root"}}},
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("schema_table_perms_db")
+    await db.execute_write(
+        "CREATE TABLE IF NOT EXISTS public_posts (id INTEGER PRIMARY KEY, title TEXT)"
+    )
+    await db.execute_write(
+        "CREATE TABLE IF NOT EXISTS employee_salaries "
+        "(id INTEGER PRIMARY KEY, ssn TEXT, salary_usd INTEGER)"
+    )
+    await db.execute_write(
+        "CREATE INDEX IF NOT EXISTS idx_employee_salaries_ssn ON employee_salaries(ssn)"
+    )
+    await db.execute_write(
+        "CREATE TRIGGER IF NOT EXISTS trg_employee_salaries "
+        "AFTER INSERT ON employee_salaries BEGIN SELECT 1; END"
+    )
+    return ds
+
+
+@pytest.mark.asyncio
+async def test_schema_table_perms_controls(schema_table_perms_ds):
+    """Sanity check: the locked down table really is denied to anonymous users."""
+    ds = schema_table_perms_ds
+    for path in (
+        "/schema_table_perms_db/employee_salaries.json",
+        "/schema_table_perms_db/employee_salaries/-/schema.json",
+        "/schema_table_perms_db/-/query.json?sql=select+*+from+employee_salaries",
+    ):
+        response = await ds.client.get(path)
+        assert response.status_code == 403, path
+    response = await ds.client.get("/schema_table_perms_db.json")
+    assert response.status_code == 200
+    assert "employee_salaries" not in response.text
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "base_url",
+    ["/-/schema", "/schema_table_perms_db/-/schema"],
+)
+@pytest.mark.parametrize("format_ext", ["json", "md", ""])
+async def test_schema_parent_views_hide_denied_tables(
+    schema_table_perms_ds, base_url, format_ext
+):
+    """
+    GHSA-926p-cw2f-643h: /-/schema and /db/-/schema must not disclose the DDL
+    of tables the actor is denied view-table on, including indexes and
+    triggers that belong to those tables.
+    """
+    url = base_url + (f".{format_ext}" if format_ext else "")
+
+    # Anonymous: allowed table visible, denied table (and its columns,
+    # index and trigger) absent
+    response = await schema_table_perms_ds.client.get(url)
+    assert response.status_code == 200
+    assert "public_posts" in response.text
+    assert "employee_salaries" not in response.text
+    assert "ssn" not in response.text
+    assert "salary_usd" not in response.text
+    assert "idx_employee_salaries_ssn" not in response.text
+    assert "trg_employee_salaries" not in response.text
+
+    # root can see everything
+    response = await schema_table_perms_ds.client.get(url, actor={"id": "root"})
+    assert response.status_code == 200
+    assert "public_posts" in response.text
+    assert "CREATE TABLE employee_salaries" in response.text
+    assert "idx_employee_salaries_ssn" in response.text
+    assert "trg_employee_salaries" in response.text
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "object_name", ["idx_employee_salaries_ssn", "trg_employee_salaries"]
+)
+@pytest.mark.parametrize("format_ext", ["json", "md", ""])
+async def test_table_schema_does_not_serve_objects_of_denied_table(
+    schema_table_perms_ds, object_name, format_ext
+):
+    """
+    Related to GHSA-926p-cw2f-643h: /db/<name>/-/schema looks up sqlite_master
+    by name without restricting to tables/views, so requesting the name of an
+    index or trigger that belongs to a denied table serves its DDL. The
+    view-table check runs against the index/trigger name, which is not a
+    restricted table, so it passes.
+    """
+    url = f"/schema_table_perms_db/{object_name}/-/schema"
+    if format_ext:
+        url += f".{format_ext}"
+    response = await schema_table_perms_ds.client.get(url)
+    assert response.status_code in (403, 404)
+    assert "employee_salaries" not in response.text
+    assert "ssn" not in response.text

tests/test_stored_queries.py

@@ -208,11 +208,12 @@ def test_custom_params(stored_write_client):
     )
 
 
-def test_stored_query_pages_no_vary_header(stored_write_client):
-    # These pages no longer embed per-cookie CSRF tokens, so they must not
-    # set Vary: Cookie - they should be cacheable across users.
-    assert "vary" not in stored_write_client.get("/data").headers
-    assert "vary" not in stored_write_client.get("/data/update_name").headers
+def test_stored_query_pages_vary_by_credentials(stored_write_client):
+    # Even without per-cookie CSRF tokens, anonymous pages must not be reused
+    # for authenticated users whose permissions or navigation can differ.
+    for path in ("/data", "/data/update_name"):
+        response = stored_write_client.get(path)
+        assert response.headers["vary"] == "Cookie, Authorization"
 
 
 def test_json_post_body(stored_write_client):

tests/test_table_api.py

@@ -4,7 +4,6 @@ import urllib
 import pytest
 
 from datasette.fixtures import generate_compound_rows, generate_sortable_rows
-from datasette.telemetry_registry import DB_QUERY, DB_QUERY_TEXT
 from datasette.utils import detect_json1, tilde_encode
 from datasette.utils.sqlite import sqlite_version
 
@@ -620,7 +619,10 @@ def test_searchmode(table_metadata, querystring, expected_rows):
             ],
         ),
         (
-            "/fixtures/searchable_view.json?_shape=arrays&_search=weasel&_fts_table=searchable_fts&_fts_pk=pk",
+            (
+                "/fixtures/searchable_view_configured_by_metadata.json"
+                "?_shape=arrays&_search=weasel&_fts_table=searchable_fts&_fts_pk=pk"
+            ),
             [[2, "terry dog", "sara weasel", "puma"]],
         ),
     ],
@@ -1202,25 +1204,11 @@ async def test_nocount(ds_client, nocount, expected_count):
     assert response.json()["count"] == expected_count
 
 
-@pytest.mark.asyncio
-async def test_nocount_nofacet_if_shape_is_object(ds_client, otel_spans):
-    # ?_extra=count and ?_facet=state each cause a count(*) query on their
-    # own. _shape=object is supposed to suppress both, so asking for them
-    # explicitly is what makes this test able to fail - a plain
-    # ?_shape=object request would never have run either query anyway.
-    response = await ds_client.get(
-        "/fixtures/facetable.json?_shape=object&_extra=count&_facet=state"
+def test_nocount_nofacet_if_shape_is_object(app_client_with_trace):
+    response = app_client_with_trace.get(
+        "/fixtures/facetable.json?_trace=1&_shape=object"
     )
-    assert response.status_code == 200
-    queries = [
-        span.attributes.get(DB_QUERY_TEXT, "")
-        for span in otel_spans.get_finished_spans()
-        if span.name == DB_QUERY
-    ]
-    # Guard: prove the request really did query the table, so the assertion
-    # below is measuring suppression rather than an empty span list.
-    assert any("from facetable" in q for q in queries), queries
-    assert not any("count(*)" in q for q in queries), queries
+    assert "count(*)" not in response.text
 
 
 @pytest.mark.asyncio
@@ -1793,3 +1781,34 @@ async def test_next_url_included_by_default(ds_client):
         data = response.json()
     assert data["next"] is None
     assert data["next_url"] is None
+
+
+@pytest.mark.asyncio
+async def test_table_through_requires_view_table_on_through_table():
+    # GHSA-53fc-rhfg-h7qp issue 3: ?_through= runs a sub-select against the
+    # caller-supplied through table, so the actor must be allowed to view it.
+    # Otherwise it is an equality oracle over any column of a denied table.
+    from datasette.app import Datasette
+
+    ds = Datasette(
+        memory=True,
+        config={"databases": {"data": {"tables": {"salaries": {"allow": False}}}}},
+    )
+    db = ds.add_memory_database("table_through_denied", name="data")
+    await db.execute_write("create table people (id integer primary key, name text)")
+    await db.execute_write(
+        "create table salaries (id integer primary key, "
+        "person_id integer references people(id), note text)"
+    )
+    await db.execute_write("insert into people values (1, 'alice'), (2, 'bob')")
+    await db.execute_write("insert into salaries values (1, 1, 'TOPSECRET-A')")
+    await ds.invoke_startup()
+
+    # Sanity: anonymous cannot read salaries directly
+    assert (await ds.client.get("/data/salaries.json")).status_code == 403
+
+    response = await ds.client.get(
+        "/data/people.json?_shape=array"
+        '&_through={"table":"salaries","column":"note","value":"TOPSECRET-A"}'
+    )
+    assert response.status_code == 403, response.text

tests/test_table_html.py

@@ -270,7 +270,8 @@ async def test_empty_search_parameter_gets_removed(ds_client):
 async def test_searchable_view_persists_fts_table(ds_client):
     # The search form should persist ?_fts_table as a hidden field
     response = await ds_client.get(
-        "/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk"
+        "/fixtures/searchable_view_configured_by_metadata"
+        "?_fts_table=searchable_fts&_fts_pk=pk"
     )
     inputs = Soup(response.text, "html.parser").find("form").find_all("input")
     hiddens = [i for i in inputs if i["type"] == "hidden"]

tests/test_table_resource_identity.py

@@ -0,0 +1,420 @@
+"""Table permission identities must agree with SQLite identifier resolution."""
+
+import uuid
+from unittest.mock import AsyncMock
+
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.default_permissions import restrictions_allow_action
+from datasette.permissions import Action, PermissionSQL, _permission_check_cache
+from datasette.resources import QueryResource, TableResource
+from datasette.utils.actions_sql import explain_permission_for_resource
+from datasette.utils.asgi import Forbidden
+from datasette.utils.permissions import gather_permission_sql_from_hooks
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["table", "view"])
+@pytest.mark.parametrize("spelling", ["Inventory", "inventory", "INVENTORY"])
+@pytest.mark.parametrize("allowed", [False, True])
+@pytest.mark.parametrize("rule_spelling", ["Inventory", "iNvEnToRy"])
+async def test_table_permission_identity(
+    kind, spelling, allowed, rule_spelling, monkeypatch
+):
+    ds = Datasette(
+        config={
+            "permissions": {"view-table": not allowed, "insert-row": not allowed},
+            "databases": {
+                "data": {
+                    "tables": {
+                        rule_spelling: {
+                            "permissions": {
+                                "view-table": allowed,
+                                "insert-row": allowed,
+                            }
+                        }
+                    }
+                }
+            },
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    cache_token = _permission_check_cache.set({})
+    try:
+        await db.execute_write(
+            "create table Inventory (id integer primary key)"
+            if kind == "table"
+            else "create view Inventory as select 1 as id"
+        )
+        await ds.invoke_startup()
+        # Identity matching needs no target-schema lookup. Derived-table
+        # permissions may still check the schema version. All spellings and
+        # API entry points should share the existing permission result cache.
+        target_execute = AsyncMock(wraps=db.execute)
+        monkeypatch.setattr(db, "execute", target_execute)
+        internal_execute = AsyncMock(wraps=ds.get_internal_database().execute)
+        monkeypatch.setattr(ds.get_internal_database(), "execute", internal_execute)
+        resource = TableResource("data", spelling)
+        assert await ds.allowed_many(
+            actions=["view-table", "insert-row"], resource=resource
+        ) == {"view-table": allowed, "insert-row": allowed}
+        assert await ds.allowed(action="view-table", resource=resource) is allowed
+        assert await ds.check_visibility(None, "view-table", resource) == (
+            allowed,
+            False,
+        )
+        if allowed:
+            await ds.ensure_permission(action="view-table", resource=resource)
+        else:
+            with pytest.raises(Forbidden):
+                await ds.ensure_permission(action="view-table", resource=resource)
+        assert resource.child == spelling  # Do not mutate caller-owned resources.
+        for variant in ("Inventory", "inventory", "INVENTORY"):
+            assert (
+                await ds.allowed(
+                    action="view-table", resource=TableResource("data", variant)
+                )
+                is allowed
+            )
+        assert internal_execute.await_count == 1
+        assert all(
+            call.args[0] == "PRAGMA schema_version"
+            for call in target_execute.await_args_list
+        )
+        assert all(key[3] == "inventory" for key in _permission_check_cache.get())
+    finally:
+        _permission_check_cache.reset(cache_token)
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_other_permission_identities_are_preserved():
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "Äpfel": {"permissions": {"view-table": False}},
+                        "Future": {"permissions": {"view-table": False}},
+                    },
+                    "queries": {
+                        "Report": {
+                            "sql": "select 1",
+                            "permissions": {"view-query": False},
+                        },
+                        "report": {
+                            "sql": "select 1",
+                            "permissions": {"view-query": True},
+                        },
+                    },
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write('create table "Äpfel" (id integer primary key)')
+        await db.execute_write('create table "äpfel" (id integer primary key)')
+        await db.execute_write("create table Report (id integer primary key)")
+        await ds.invoke_startup()
+        # SQLite folds ASCII identifier casing, not Unicode casing.
+        for name, expected in [
+            ("ÄPFEL", False),
+            ("äPFEL", True),
+            ("Future", False),
+            ("future", False),
+        ]:
+            assert (
+                await ds.allowed(
+                    action="view-table", resource=TableResource("data", name)
+                )
+                is expected
+            )
+        # Query names remain case-sensitive even when a table has the same name.
+        for name, expected in [("Report", False), ("report", True)]:
+            assert (
+                await ds.allowed(
+                    action="view-query", resource=QueryResource("data", name)
+                )
+                is expected
+            )
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("allow", [True, False, {"id": "reader"}])
+async def test_table_listings_and_explanations(allow):
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "inventory": {"permissions": {"view-table": allow}},
+                    }
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await db.execute_write("create view InventoryView as select id from Inventory")
+        await ds.invoke_startup()
+        for actor in (None, {"id": "reader"}):
+            expected = allow is True or (isinstance(allow, dict) and actor == allow)
+            explanation = await explain_permission_for_resource(
+                datasette=ds,
+                actor=actor,
+                action="view-table",
+                parent="data",
+                child="INVENTORY",
+            )
+            assert explanation["allowed"] is expected
+            assert explanation["winning_scope"] == "resource"
+            assert any(
+                "data/inventory" in rule["reason"]
+                for rule in explanation["matched_rules"]
+            )
+            page = await ds.allowed_resources(
+                "view-table",
+                actor,
+                parent="data",
+                include_is_private=True,
+                include_reasons=True,
+                limit=1,
+            )
+            resources = [resource async for resource in page.all()]
+            matching = [r for r in resources if r.child == "Inventory"]
+            assert bool(matching) is expected
+            assert len(matching) <= 1
+            if matching:
+                assert matching[0].private is isinstance(allow, dict)
+            assert any(r.child == "InventoryView" for r in resources)
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("deny_first", [True, False])
+async def test_case_variant_rules_deny_wins(deny_first):
+    rules = [("inventory", False), ("INVENTORY", True)]
+    if not deny_first:
+        rules.reverse()
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        name: {"permissions": {"view-table": allow}}
+                        for name, allow in rules
+                    }
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await ds.invoke_startup()
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "Inventory")
+        )
+        assert not (
+            await ds.allowed_resources(
+                "view-table", parent="data", include_is_private=True
+            )
+        ).resources
+        explanation = await explain_permission_for_resource(
+            datasette=ds,
+            actor=None,
+            action="view-table",
+            parent="data",
+            child="Inventory",
+        )
+        assert not explanation["allowed"]
+        assert any(
+            rule["effect"] == "allow" and not rule["decisive"]
+            for rule in explanation["matched_rules"]
+        )
+        assert any(
+            rule["effect"] == "deny" and rule["decisive"]
+            for rule in explanation["matched_rules"]
+        )
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("config_style", ["allow", "permissions"])
+@pytest.mark.parametrize("allowed", [True, False])
+async def test_case_variant_token_restrictions(config_style, allowed):
+    table_config = (
+        {"allow": allowed}
+        if config_style == "allow"
+        else {"permissions": {"view-table": allowed}}
+    )
+    ds = Datasette(
+        config={"databases": {"data": {"tables": {"Inventory": table_config}}}}
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    actor = {"id": "reader", "_r": {"r": {"data": {"inventory": ["vt"]}}}}
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await ds.invoke_startup()
+        assert restrictions_allow_action(
+            ds, actor["_r"], "view-table", ("data", "INVENTORY")
+        )
+        assert not restrictions_allow_action(
+            ds, actor["_r"], "view-table", ("Data", "Inventory")
+        )
+        assert (
+            await ds.allowed(
+                action="view-table",
+                resource=TableResource("data", "INVENTORY"),
+                actor=actor,
+            )
+            is allowed
+        )
+        page = await ds.allowed_resources("view-table", actor, parent="data")
+        assert [(r.parent, r.child) for r in page.resources] == (
+            [("data", "Inventory")] if allowed else []
+        )
+        explanation = await explain_permission_for_resource(
+            datasette=ds,
+            actor=actor,
+            action="view-table",
+            parent="data",
+            child="Inventory",
+        )
+        assert explanation["restriction_allowed"]
+        assert explanation["allowed"] is allowed
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_plugin_restriction_intersection_and_dependencies():
+    class Plugin:
+        @hookimpl
+        def register_actions(self, datasette):
+            return [
+                Action(
+                    name="inspect-inventory",
+                    description="Inspect inventory",
+                    resource_class=TableResource,
+                    also_requires="view-table",
+                )
+            ]
+
+        @hookimpl
+        def permission_resources_sql(self, action):
+            if action not in ("view-table", "inspect-inventory"):
+                return None
+            return [
+                PermissionSQL(
+                    sql="SELECT 'data' AS parent, 'INVENTORY' AS child, 1 AS allow, 'inventory grant' AS reason",
+                    restriction_sql="SELECT 'data' AS parent, 'inventory' AS child",
+                ),
+                PermissionSQL(
+                    restriction_sql="SELECT 'data' AS parent, 'InVeNtOrY' AS child"
+                ),
+            ]
+
+    ds = Datasette(default_deny=True)
+    ds.pm.register(Plugin(), name="identity-test")
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await db.execute_write("create table Other (id integer primary key)")
+        await ds.invoke_startup()
+        for action in ("view-table", "inspect-inventory"):
+            assert await ds.allowed(
+                action=action, resource=TableResource("data", "Inventory")
+            )
+            assert not await ds.allowed(
+                action=action, resource=TableResource("data", "Other")
+            )
+            resources = (
+                await ds.allowed_resources(
+                    action, parent="data", include_is_private=True
+                )
+            ).resources
+            assert [r.child for r in resources] == ["Inventory"]
+            explanation = await explain_permission_for_resource(
+                datasette=ds,
+                actor=None,
+                action=action,
+                parent="data",
+                child="Inventory",
+            )
+            assert explanation["allowed"]
+            assert all(item["allowed"] for item in explanation["restrictions"])
+    finally:
+        ds.pm.unregister(name="identity-test")
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_shared_plugin_rule_keeps_query_identity_and_original_sql():
+    shared = PermissionSQL(
+        sql="SELECT 'data' AS parent, 'Inventory' AS child, 0 AS allow, 'shared deny' AS reason"
+    )
+    original_sql = shared.sql
+
+    class Plugin:
+        @hookimpl
+        def permission_resources_sql(self, action):
+            if action in ("view-table", "view-query"):
+                return shared
+
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "queries": {
+                        "Inventory": "select 1",
+                        "inventory": "select 1",
+                    }
+                }
+            }
+        }
+    )
+    ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    ds.pm.register(Plugin(), name="identity-test")
+    try:
+        await ds.invoke_startup()
+        for _ in range(2):
+            await gather_permission_sql_from_hooks(
+                datasette=ds, actor=None, action="view-table"
+            )
+        assert shared.sql == original_sql
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "inventory")
+        )
+        assert await ds.allowed(
+            action="view-query", resource=QueryResource("data", "inventory")
+        )
+        assert not await ds.allowed(
+            action="view-query", resource=QueryResource("data", "Inventory")
+        )
+        assert await ds.allowed(
+            action="view-table", resource=TableResource("Data", "Inventory")
+        )
+        assert restrictions_allow_action(
+            ds,
+            {"r": {"data": {"Inventory": ["vq"]}}},
+            "view-query",
+            ("data", "Inventory"),
+        )
+        assert not restrictions_allow_action(
+            ds,
+            {"r": {"data": {"Inventory": ["vq"]}}},
+            "view-query",
+            ("data", "inventory"),
+        )
+    finally:
+        ds.pm.unregister(name="identity-test")
+        ds.close()

tests/test_telemetry_registry.py

@@ -1,431 +0,0 @@
-"""
-Two-way conformance between `datasette/telemetry_registry.py` and what
-Datasette actually emits.
-
-This is the test that makes the generated documentation trustworthy. cog
-guarantees the docs match the registry; this guarantees the registry matches
-the code. Without it, both could agree with each other and be wrong.
-
-It checks both directions, and the second one is the one nothing else catches:
-
-- **emitted but not registered** - instrumentation was added without
-  documenting it, so the reference page silently omits it.
-- **registered but never emitted** - the reference page describes a span or
-  attribute that no longer exists, which is worse than omitting it, because a
-  reader will build a dashboard on it.
-
-Both of those directions compare the code against the registry. Neither can
-catch a *rename*, because the call sites now take their names from the
-registry - move `DB_NAMESPACE` to `"db.namespace2"` and code and registry
-still agree with each other, while every existing dashboard breaks. So the
-literal names live here too, spelled out, and are asserted against both the
-registry and the wire. That is the one comparison in this file that is not
-made against a value derived from the registry itself.
-"""
-
-import itertools
-
-import pytest
-import pytest_asyncio
-
-pytest.importorskip("opentelemetry.sdk")
-
-from opentelemetry.trace import SpanKind
-
-from datasette import hookimpl
-from datasette import telemetry_registry as reg
-from datasette.app import Datasette
-from datasette.database import QueryInterrupted
-from datasette.utils.sqlite import sqlite3
-
-# The names as they appear on the wire, written out rather than read from the
-# registry. If a change to the registry makes one of these fail, that change
-# is renaming something a user's dashboards and saved queries depend on -
-# which is a decision to take deliberately, here, not a line to re-derive.
-EXPECTED_ATTRIBUTES = {
-    "db.query": {
-        "db.system",
-        "db.namespace",
-        "db.query.text",
-        "db.operation.name",
-        "db.collection.name",
-        "datasette.param_count",
-        "datasette.param_sets",
-        "datasette.time_limit_ms",
-        "datasette.rows_returned",
-        "datasette.truncated",
-        "datasette.interrupted",
-        "datasette.sql_error_suppressed",
-        "datasette.executescript",
-        "datasette.executemany",
-    },
-    "db.query.execute": set(),
-    "db.write.queue_wait": set(),
-    "db.write.execute": {
-        "datasette.isolated_connection",
-        "datasette.transaction",
-    },
-    "datasette.startup": set(),
-}
-EXPECTED_SPANS = set(EXPECTED_ATTRIBUTES)
-
-# The HTTP request span is handled separately because its name is composed at
-# runtime - the request method, then the route it matched - so there is no
-# fixed string to pin it to. What can still be pinned, and is what a dashboard
-# depends on, is the shape of that name and the attribute keys.
-#
-# The route half is deliberately not spelled out as a literal: it is a core
-# route regex, and pinning those here would make an unrelated routing change
-# fail the telemetry conformance test. What is pinned instead is that the name
-# is exactly the method, a space, and the span's own `http.route` value - the
-# `{method} {route}` shape semantic conventions specify. The workload below
-# only issues GETs, so a change that stopped clamping the method, or that
-# started naming the span after the path, fails here.
-EXPECTED_HTTP_SPAN_NAME = "{http.request.method} {http.route}"
-EXPECTED_HTTP_METHOD_NAMES = {"GET"}
-EXPECTED_HTTP_ATTRIBUTES = {
-    "http.request.method",
-    "http.route",
-    "url.path",
-    "url.scheme",
-    "server.address",
-    "user_agent.original",
-    "http.response.status_code",
-    "error.type",
-}
-
-# The registry's own name for the request span is that template, not anything
-# that appears on the wire.
-EXPECTED_REGISTRY_ATTRIBUTES = dict(
-    EXPECTED_ATTRIBUTES, **{EXPECTED_HTTP_SPAN_NAME: EXPECTED_HTTP_ATTRIBUTES}
-)
-EXPECTED_REGISTRY_NAMES = set(EXPECTED_REGISTRY_ATTRIBUTES)
-
-# Named in-memory databases are shared-cache, so two Datasette instances using
-# the same name share one SQLite database - and the second `create table`
-# fails. Every workload below therefore gets its own name.
-_names = itertools.count()
-
-
-def _unique(prefix):
-    return f"{prefix}{next(_names)}"
-
-
-class _BoomPlugin:
-    """
-    A route that raises.
-
-    `error.type` on the request span is only ever set by a 5xx, and nothing
-    in Datasette returns one on a healthy instance - `route_path` converts
-    exceptions into a 500 itself, so the workload has to supply the
-    exception.
-    """
-
-    __name__ = "TelemetryRegistryBoomPlugin"
-
-    @hookimpl
-    def register_routes(self):
-        return [(r"^/-/telemetry-registry-boom$", lambda: 1 / 0)]
-
-
-async def exercise():
-    """
-    Drive enough of Datasette to emit every span and attribute the registry
-    claims exists.
-
-    Each call is here because it is the only thing that produces some span or
-    attribute - see the comments. If you add instrumentation on a path this
-    does not reach, add the path rather than loosening the assertions.
-
-    Returns the instance so the caller can close it; startup happens inside
-    so that the `datasette.startup` span lands in the collected set.
-    """
-    name = _unique("registry")
-    ds = Datasette(memory=True)
-    ds.add_memory_database(name)
-    # datasette.startup - and the internal catalog work nested under it
-    await ds.invoke_startup()
-    db = ds.get_database(name)
-
-    # Writes: db.write.queue_wait, db.write.execute, db.query
-    await db.execute_write("create table t (id integer primary key, v text)")
-    # datasette.executemany, datasette.param_sets
-    await db.execute_write_many(
-        "insert into t (id, v) values (?, ?)", [[i, f"v{i}"] for i in range(30)]
-    )
-    # datasette.executescript
-    await db.execute_write_script("create table t2 (id integer); drop table t2;")
-    # datasette.transaction=False - VACUUM cannot run inside a transaction
-    await db.execute_write("vacuum", transaction=False)
-    # datasette.isolated_connection=True
-    await db.execute_isolated_fn(lambda conn: conn.execute("select 1").fetchone())
-
-    # Reads: db.query.execute, datasette.rows_returned, datasette.truncated,
-    # datasette.param_count, datasette.time_limit_ms
-    await db.execute("select * from t where id > :n", {"n": 5})
-    await db.execute("select * from t", truncate=True)
-
-    # datasette.sql_error_suppressed - the caller is probing and treats
-    # failure as an expected answer
-    with pytest.raises(sqlite3.OperationalError):
-        await db.execute("select nope from t", log_sql_errors=False)
-
-    # datasette.interrupted - only ever set when a query exceeds its time
-    # limit, so the workload has to force one rather than exempt it. An
-    # unbounded recursive CTE cannot finish, so 1ms is always exceeded.
-    with pytest.raises(QueryInterrupted):
-        await db.execute(
-            "with recursive c(x) as (select 0 union all select x+1 from c) "
-            "select * from c",
-            custom_time_limit=1,
-        )
-
-    # db.collection.name - set only by views that already know their table.
-    # These requests are also what produces the HTTP request span and its
-    # http.request.method / url.path / url.scheme / server.address /
-    # user_agent.original / http.response.status_code attributes.
-    assert (await ds.client.get(f"/{name}/t?_facet=v")).status_code == 200
-    assert (await ds.client.get(f"/{name}/t/1.json")).status_code == 200
-
-    # error.type on the request span, which only a 5xx sets
-    ds.pm.register(_BoomPlugin(), name="telemetry-registry-boom")
-    try:
-        response = await ds.client.get("/-/telemetry-registry-boom")
-        assert response.status_code == 500
-    finally:
-        ds.pm.unregister(name="telemetry-registry-boom")
-    return ds
-
-
-@pytest_asyncio.fixture
-async def emitted(otel_spans):
-    """
-    Every (span name, span kind, attributes) triple a broad workload emits.
-
-    The kind is carried because the request span's name is composed at
-    runtime, so `span_for()` resolves it by kind instead. The attributes are
-    carried as a mapping rather than a set of keys because the request span's
-    name has to be checked against its own `http.route` value.
-    """
-    # otel_spans has already cleared the exporter, and nothing is cleared
-    # after this point: the workload's own startup emits datasette.startup.
-    ds = await exercise()
-    spans = otel_spans.get_finished_spans()
-    assert spans, "no spans captured - the fixture is not exercising anything"
-    # str() because span.name is the registry's SpanName instance, and a set
-    # of those would compare equal to literals but read confusingly in a
-    # failure message.
-    collected = tuple(
-        (
-            str(span.name),
-            span.kind,
-            {str(key): value for key, value in (span.attributes or {}).items()},
-        )
-        for span in spans
-    )
-    ds.close()
-    return collected
-
-
-def _partition(emitted):
-    "The statically named spans, and the dynamically named request spans."
-    static = [record for record in emitted if record[1] is not SpanKind.SERVER]
-    server = [record for record in emitted if record[1] is SpanKind.SERVER]
-    return static, server
-
-
-def _keys_by_span(records):
-    by_span = {}
-    for name, _kind, attributes in records:
-        by_span.setdefault(name, set()).update(attributes)
-    return by_span
-
-
-@pytest.mark.asyncio
-async def test_workload_emits_exactly_the_expected_names(emitted):
-    """
-    The wire format, pinned to literals.
-
-    Not derived from the registry, so this is what catches a rename that the
-    registry and the call sites make together.
-    """
-    static, server = _partition(emitted)
-    by_span = _keys_by_span(static)
-    assert set(by_span) == EXPECTED_SPANS
-    assert by_span == EXPECTED_ATTRIBUTES
-
-    assert server, "the workload made HTTP requests but no SERVER span was emitted"
-    union = set()
-    methods = set()
-    for name, _kind, attributes in server:
-        union |= set(attributes)
-        route = attributes.get("http.route")
-        # Every request in the workload matches a route, so every one of these
-        # names must be `{method} {route}`. A 404 would be a bare method - the
-        # http_route tests cover that case with a real request.
-        assert route, f"the request span {name!r} carries no http.route"
-        method, _, name_route = name.partition(" ")
-        assert name_route == route, (
-            f"the request span is named {name!r}, which is not the "
-            f"`{{method}} {{route}}` of {method!r} and {route!r}"
-        )
-        methods.add(method)
-    assert methods == EXPECTED_HTTP_METHOD_NAMES
-    assert union == EXPECTED_HTTP_ATTRIBUTES
-
-
-def test_registry_matches_the_expected_names():
-    "The other half of the rename check: the registry against the same literals."
-    assert {str(span) for span in reg.SPANS} == EXPECTED_REGISTRY_NAMES
-    for span in reg.SPANS:
-        assert {
-            str(attribute) for attribute in span.attributes
-        } == EXPECTED_REGISTRY_ATTRIBUTES[str(span)], f"{span} attributes have drifted"
-
-
-@pytest.mark.asyncio
-async def test_every_emitted_span_is_registered(emitted):
-    "A span added without a registry entry would be missing from the docs."
-    unregistered = sorted(
-        {name for name, kind, _ in emitted if reg.span_for(name, kind) is None}
-    )
-    assert (
-        not unregistered
-    ), f"these spans are emitted but not in telemetry_registry.SPANS: {unregistered}"
-
-
-@pytest.mark.asyncio
-async def test_every_emitted_attribute_is_registered(emitted):
-    "An attribute added without a registry entry would be missing from the docs."
-    unregistered = sorted(
-        {
-            f"{name} -> {key}"
-            for name, kind, keys in emitted
-            for key in keys
-            if not reg.attribute_allowed(reg.span_for(name, kind), key)
-        }
-    )
-    assert (
-        not unregistered
-    ), "these span attributes are emitted but not registered: " + ", ".join(
-        unregistered
-    )
-
-
-@pytest.mark.asyncio
-async def test_every_registered_span_is_emitted(emitted):
-    """
-    The direction nothing else catches: the docs must not describe a span that
-    no longer exists.
-    """
-    # By identity, not by name: a dynamic entry's own string never appears on
-    # the wire, so comparing strings would be comparing the wrong things.
-    resolved = {id(reg.span_for(name, kind)) for name, kind, _ in emitted}
-    missing = sorted(str(span) for span in reg.SPANS if id(span) not in resolved)
-    assert not missing, (
-        f"these spans are documented but never emitted by the workload: {missing}. "
-        "Either the instrumentation was removed, or exercise() no longer reaches it."
-    )
-
-
-@pytest.mark.asyncio
-async def test_every_registered_attribute_is_emitted(emitted):
-    """
-    Every registered attribute, optional or not, must actually be set at least
-    once by the workload.
-
-    `optional` describes whether a reader should expect it on every span, not
-    whether the code still sets it - so an attribute deleted from the code but
-    left in the docs has to fail here even when it is marked optional. If a
-    new attribute only appears in some rare case, extend exercise() to reach
-    that case.
-    """
-    by_entry = {}
-    for name, kind, keys in emitted:
-        entry = reg.span_for(name, kind)
-        if entry is not None:
-            by_entry.setdefault(id(entry), set()).update(keys)
-    missing = []
-    for span in reg.SPANS:
-        emitted_keys = by_entry.get(id(span), set())
-        for attribute in span.attributes:
-            if attribute not in emitted_keys:
-                missing.append(f"{span} -> {attribute}")
-    assert not missing, (
-        "these attributes are documented but never emitted by the workload: "
-        + ", ".join(sorted(missing))
-    )
-
-
-def test_registry_has_no_duplicate_names():
-    assert len(set(reg.SPANS)) == len(reg.SPANS)
-    for span in reg.SPANS:
-        assert len(set(span.attributes)) == len(
-            span.attributes
-        ), f"{span} lists an attribute twice"
-
-
-def test_registry_entries_are_documented():
-    "Every entry carries a description - the docs are generated from these."
-    for span in reg.SPANS:
-        assert span.description.strip(), f"{span} has no description"
-        for attribute in span.attributes:
-            assert attribute.description.strip(), f"{span} -> {attribute} has none"
-
-
-def test_registry_entries_are_usable_as_plain_strings():
-    "The str subclassing is the whole reason call sites need no wrapper API."
-    assert isinstance(reg.DB_QUERY, str)
-    assert isinstance(reg.DB_NAMESPACE, str)
-    assert reg.DB_QUERY == "db.query"
-    assert reg.DB_NAMESPACE == "db.namespace"
-    assert f"{reg.DB_QUERY}.execute" == "db.query.execute"
-
-
-def test_dynamic_span_lookup():
-    """
-    `dynamic=True` matching, which is how the request span resolves.
-
-    The last two assertions are the ones worth having: a dynamic entry must
-    not swallow a span that does have a registered name, and must not match at
-    all when the caller supplies no kind - otherwise every unregistered span
-    in the suite would silently resolve to the request span and the
-    emitted-but-not-registered direction would stop catching anything.
-    """
-    assert reg.span_for("GET", SpanKind.SERVER) is reg.HTTP_REQUEST
-    assert reg.span_for("POST /^/(?P<database>[^/]+)$", SpanKind.SERVER) is (
-        reg.HTTP_REQUEST
-    )
-    assert reg.span_for("GET") is None
-    assert reg.span_for("anything at all", SpanKind.INTERNAL) is None
-    assert reg.span_for("db.query", SpanKind.SERVER) is reg.DB_QUERY
-
-
-def test_span_and_attribute_lookup():
-    assert reg.span_for("db.query") is reg.DB_QUERY
-    assert reg.span_for("datasette.startup") is reg.STARTUP
-    assert reg.span_for("not.a.datasette.span") is None
-    assert reg.attribute_allowed(reg.DB_QUERY, "db.namespace")
-    assert not reg.attribute_allowed(reg.DB_QUERY, "db.namespace.extra")
-    assert not reg.attribute_allowed(reg.DB_QUERY, "datasette.isolated_connection")
-    assert not reg.attribute_allowed(None, "db.namespace")
-
-
-def test_prefix_span_lookup():
-    """
-    `prefix=True` matching, exercised directly.
-
-    Phase 1 registers no prefix spans, so without this the branch in
-    `span_for()` would be untested code that the conformance tests silently
-    never reach.
-    """
-    hook = reg.SpanName("datasette.hook.", "A hypothetical span family", prefix=True)
-    original = reg.SPANS
-    reg.SPANS = original + (hook,)
-    try:
-        assert reg.span_for("datasette.hook.render_cell") is hook
-        assert reg.span_for("datasette.hook.anything") is hook
-        assert reg.span_for("datasette.hookish") is None
-        assert reg.span_for("db.query") is reg.DB_QUERY
-    finally:
-        reg.SPANS = original

tests/test_tracer.py

@@ -0,0 +1,98 @@
+import pytest
+
+from .fixtures import make_app_client
+
+
+@pytest.mark.parametrize("trace_debug", (True, False))
+def test_trace(trace_debug):
+    with make_app_client(settings={"trace_debug": trace_debug}) as client:
+        response = client.get("/fixtures/simple_primary_key.json?_trace=1")
+        assert response.status == 200
+
+    data = response.json
+    if not trace_debug:
+        assert "_trace" not in data
+        return
+
+    assert "_trace" in data
+    trace_info = data["_trace"]
+    assert isinstance(trace_info["request_duration_ms"], float)
+    assert isinstance(trace_info["sum_trace_duration_ms"], float)
+    assert isinstance(trace_info["num_traces"], int)
+    assert isinstance(trace_info["traces"], list)
+    traces = trace_info["traces"]
+    assert len(traces) == trace_info["num_traces"]
+    for trace in traces:
+        assert isinstance(trace["type"], str)
+        assert isinstance(trace["start"], float)
+        assert isinstance(trace["end"], float)
+        assert trace["duration_ms"] == (trace["end"] - trace["start"]) * 1000
+        assert isinstance(trace["traceback"], list)
+        assert isinstance(trace["database"], str)
+        assert isinstance(trace["sql"], str)
+        assert isinstance(trace.get("params"), (list, dict, None.__class__))
+
+    sqls = [trace["sql"] for trace in traces if "sql" in trace]
+    # There should be SQL statements from request handling in the trace.
+    # Note: CREATE TABLE, INSERT OR REPLACE, executescript, and executemany
+    # are not expected here because internal tables are now created and
+    # populated during invoke_startup(), before the request is traced.
+    assert any(sql.startswith("select ") for sql in sqls), "No select statements traced"
+
+
+def test_trace_silently_fails_for_large_page():
+    # Max HTML size is 256KB
+    with make_app_client(settings={"trace_debug": True}) as client:
+        # Small response should have trace
+        small_response = client.get("/fixtures/simple_primary_key.json?_trace=1")
+        assert small_response.status == 200
+        assert "_trace" in small_response.json
+
+        # Big response should not
+        big_response = client.get(
+            "/fixtures/-/query.json",
+            params={"_trace": 1, "sql": "select zeroblob(1024 * 256)"},
+        )
+        assert big_response.status == 200
+        assert "_trace" not in big_response.json
+
+
+def test_trace_query_errors():
+    with make_app_client(settings={"trace_debug": True}) as client:
+        response = client.get(
+            "/fixtures/-/query.json",
+            params={"_trace": 1, "sql": "select * from non_existent_table"},
+        )
+        assert response.status == 400
+
+    data = response.json
+    assert "_trace" in data
+    trace_info = data["_trace"]
+    assert trace_info["traces"][-1]["error"] == "no such table: non_existent_table"
+
+
+@pytest.mark.asyncio
+async def test_trace_child_tasks_resets_contextvar_on_exception():
+    from datasette import tracer
+
+    before = tracer.trace_task_id.get()
+    with pytest.raises(ValueError), tracer.trace_child_tasks():
+        assert tracer.trace_task_id.get() is not None
+        raise ValueError("simulated error")
+    # The contextvar must be reset even though the block raised
+    assert tracer.trace_task_id.get() == before
+
+
+def test_trace_parallel_queries():
+    with make_app_client(settings={"trace_debug": True}) as client:
+        response = client.get("/parallel-queries?_trace=1")
+        assert response.status == 200
+
+    data = response.json
+    assert data["one"] == 1
+    assert data["two"] == 2
+    trace_info = data["_trace"]
+    traces = [trace for trace in trace_info["traces"] if "sql" in trace]
+    one, two = traces
+    # "two" should have started before "one" ended
+    assert two["start"] < one["end"]

tests/test_utils.py

@@ -16,6 +16,7 @@ from datasette.app import Datasette
 from datasette.utils.asgi import Request
 from datasette.utils.sqlite import (
     sqlite3,
+    sqlite_derived_table_dependencies,
     sqlite_hidden_table_names,
     sqlite_table_type,
     supports_returning,
@@ -369,6 +370,46 @@ def test_sqlite_hidden_table_names_hides_multiline_content_fts_table():
         conn.close()
 
 
+def test_sqlite_derived_table_dependencies():
+    conn = utils.sqlite3.connect(":memory:")
+    try:
+        conn.executescript("""
+            create table docs(id integer primary key, body text);
+            create virtual table external_fts5 using fts5(
+                body, content='docs', content_rowid='id'
+            );
+            create virtual table internal_fts5 using fts5(body);
+            create virtual table contentless_fts5 using fts5(body, content='');
+            create virtual table external_fts4 using fts4(body, content="docs");
+            create virtual table internal_fts4 using fts4(body);
+            create virtual table contentless_fts4 using fts4(body, content="");
+            create table [docs, archive](body text);
+            create virtual table commented_fts5 using fts5(
+                body, tokenize='porter unicode61',
+                /* Comments and commas in quoted values must not confuse parsing. */
+                content='docs, archive'
+            );
+            create virtual table boxes using rtree(id, minx, maxx, miny, maxy);
+        """)
+
+        dependencies = sqlite_derived_table_dependencies(conn)
+
+        assert dependencies["external_fts5"] == "docs"
+        assert dependencies["external_fts4"] == "docs"
+        assert dependencies["commented_fts5"] == "docs, archive"
+        assert "contentless_fts5" not in dependencies
+        assert "contentless_fts4" not in dependencies
+        assert dependencies["internal_fts5_content"] == "internal_fts5"
+        assert dependencies["internal_fts4_content"] == "internal_fts4"
+        assert dependencies["external_fts5_data"] == "external_fts5"
+        assert dependencies["external_fts4_segments"] == "external_fts4"
+        assert dependencies["boxes_node"] == "boxes"
+        assert dependencies["boxes_parent"] == "boxes"
+        assert dependencies["boxes_rowid"] == "boxes"
+    finally:
+        conn.close()
+
+
 @pytest.mark.parametrize(
     "url,expected",
     [
@@ -852,13 +893,13 @@ def test_truncate_url(url, length, expected):
         ),
         (
             [
-                ("settings.template_debug", "true"),
+                ("settings.trace_debug", "true"),
                 ("plugins.datasette-ripgrep.path", "/etc"),
-                ("settings.template_debug", "false"),
+                ("settings.trace_debug", "false"),
             ],
             {
                 "settings": {
-                    "template_debug": False,
+                    "trace_debug": False,
                 },
                 "plugins": {
                     "datasette-ripgrep": {

tests/test_utils_sql_analysis.py

@@ -439,7 +439,7 @@ def test_analyze_attached_database_tables(conn):
     }
 
 
-def test_analyze_clears_authorizer_on_error():
+def test_analyze_disables_authorizer_on_error():
     class FakeConnection:
         def __init__(self):
             self.authorizers = []
@@ -455,4 +455,5 @@ def test_analyze_clears_authorizer_on_error():
     with pytest.raises(sqlite3.OperationalError):
         analyze_sql_tables(conn, "bad SQL")
 
-    assert conn.authorizers[-1] is None
+    final_authorizer = conn.authorizers[-1]
+    assert final_authorizer is None or final_authorizer() == sqlite3.SQLITE_OK