Screenshots for reusable modal dialogs docs, plus just shots (#2958)

docs/shots.yml defines documentation screenshots for shot-scraper

"Just shots" runs them.

.github/workflows/deploy-latest.yml

@@ -14,24 +14,46 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
+    - name: Check deployment prerequisites
+      id: deployment-prerequisites
+      env:
+        GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+        LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
+      run: |
+        missing=()
+        for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do
+          if [[ -z "${!variable:-}" ]]; then
+            missing+=("$variable")
+          fi
+        done
+        if (( ${#missing[@]} )); then
+          echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}"
+          echo "available=false" >> "$GITHUB_OUTPUT"
+        else
+          echo "available=true" >> "$GITHUB_OUTPUT"
+        fi
     - name: Check out datasette
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/checkout@v7
     - name: Set up Python
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/setup-python@v6
       with:
         python-version: "3.13"
         cache: pip
     - name: Install Python dependencies
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         python -m pip install --upgrade pip
         python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1
+        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
     - name: Run tests
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
     - name: Build fixtures.db and other files needed to deploy the demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |-
         python tests/fixtures.py \
           fixtures.db \
@@ -39,14 +61,18 @@ jobs:
           fixtures-metadata.json \
           plugins \
           --extra-db-filename extra_database.db
+        # Package the config with the plugins, excluding test-only plugin secrets
+        # that reference temporary files outside the deployed container.
+        jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json
     - name: Build docs.db
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
     - name: Set up the alternate-route demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         echo '
         from datasette import hookimpl
@@ -58,6 +84,7 @@ jobs:
         ' > plugins/alternative_route.py
         cp fixtures.db fixtures2.db
     - name: And the counters writable stored query demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         cat > plugins/counters.py <<EOF
         from datasette import hookimpl
@@ -97,12 +124,15 @@ jobs:
     #     cat metadata.json
     - id: auth
       name: Authenticate to Google Cloud
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/auth@v3
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}
     - name: Set up Cloud SDK
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/setup-gcloud@v3
     - name: Deploy to Cloud Run
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       env:
         LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
       run: |-
@@ -117,16 +147,16 @@ jobs:
             --plugins-dir=plugins \
             --branch=$GITHUB_SHA \
             --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
+            --extra-options="--config plugins/fixtures-config.json --setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
             --install 'datasette-ephemeral-tables>=0.2.2' \
             --service "datasette-latest$SUFFIX" \
             --secret $LATEST_DATASETTE_SECRET
-    - name: Deploy to docs as well (only for main)
+    - name: Upload latest documentation database to S3 (only for main)
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
       run: |-
-        # Deploy docs.db to a different service
+        # Keep development documentation separate from the stable release database.
-        datasette publish cloudrun docs.db \
+        s3-credentials put-object datasette-docs latest/docs.db docs.db \
-            --branch=$GITHUB_SHA \
+          --content-type application/octet-stream
-            --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1" \
-            --service=datasette-docs-latest

.github/workflows/playwright.yml

@@ -2,9 +2,15 @@ name: Playwright
 
 on:
   push:
+    branches:
+    - main
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 permissions:
   contents: read
 

.github/workflows/prettier.yml

@@ -1,6 +1,15 @@
 name: Check JavaScript for conformance with Prettier
 
-on: [push]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read

.github/workflows/publish.yml

@@ -2,7 +2,7 @@ name: Publish Python Package
 
 on:
   release:
-    types: [created]
+    types: [published]
 
 permissions:
   contents: read
@@ -51,6 +51,8 @@ jobs:
     - name: Publish
       uses: pypa/gh-action-pypi-publish@release/v1
 
+  # After the first non-prerelease 1.0 release, disable this job on 0.65.x,
+  # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs.
   deploy_static_docs:
     runs-on: ubuntu-latest
     needs: [deploy]
@@ -66,26 +68,20 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1
+        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
     - name: Build docs.db
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
-    - id: auth
+    - name: Upload stable documentation database to S3
-      name: Authenticate to Google Cloud
+      env:
-      uses: google-github-actions/auth@v2
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
-      with:
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
-        credentials_json: ${{ secrets.GCP_SA_KEY }}
-    - name: Set up Cloud SDK
-      uses: google-github-actions/setup-gcloud@v3
-    - name: Deploy stable-docs.datasette.io to Cloud Run
       run: |-
-        gcloud config set run/region us-central1
+        s3-credentials put-object datasette-docs docs.db docs.db \
-        gcloud config set project datasette-222320
+          --content-type application/octet-stream
-        datasette publish cloudrun docs.db \
-            --service=datasette-docs-stable
 
   deploy_docker:
     runs-on: ubuntu-latest

.github/workflows/spellcheck.yml

@@ -1,6 +1,15 @@
 name: Check spelling in documentation
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read

.github/workflows/test-coverage.yml

@@ -1,40 +0,0 @@
-name: Calculate test coverage
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out datasette
-      uses: actions/checkout@v7
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version: '3.12'
-        cache: 'pip'
-        cache-dependency-path: '**/pyproject.toml'
-    - name: Install Python dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install . --group dev
-        python -m pip install pytest-cov
-    - name: Run tests
-      run: |-
-        ls -lah
-        cat .coveragerc
-        pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x
-        ls -lah
-    - name: Upload coverage report
-      uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.xml

.github/workflows/test-pyodide.yml

@@ -2,9 +2,15 @@ name: Test in Pyodide with shot-scraper
 
 on:
   push:
+    branches:
+    - main
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 permissions:
   contents: read
 

.github/workflows/test-sqlite-support.yml

@@ -1,6 +1,15 @@
 name: Test SQLite versions
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -12,10 +21,10 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest]
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.13"]
         sqlite-version: [
           #"3", # latest version
-          "3.46",
+          #"3.46",
           #"3.45",
           #"3.27",
           #"3.26",

.github/workflows/test.yml

@@ -1,6 +1,15 @@
 name: Test
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -11,16 +20,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
+        include:
+        - python-version: "3.14"
+          coverage: true
     steps:
     - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v6
+      uses: actions/setup-python@v7
       with:
         python-version: ${{ matrix.python-version }}
         allow-prereleases: true
         cache: pip
         cache-dependency-path: pyproject.toml
+        check-latest: true
     - name: Build extension for --load-extension test
       run: |-
         (cd tests && gcc ext.c -fPIC -shared -o ext.so)
@@ -28,12 +41,27 @@ jobs:
       run: |
         pip install . --group dev
         pip freeze
+    - name: Install pytest-cov
+      if: ${{ matrix.coverage }}
+      run: pip install pytest-cov
     - name: Run tests
       run: |
-        pytest -n auto -m "not serial"
+        if [ "${{ matrix.coverage }}" = "true" ]; then
-        pytest -m "serial"
+          COV="--cov=datasette --cov-config=.coveragerc"
+          pytest -n auto -m "not serial" $COV --cov-report=
+          pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term
+        else
+          pytest -n auto -m "not serial"
+          pytest -m "serial"
+        fi
         # And the test that exceeds a localhost HTTPS server
         tests/test_datasette_https_server.sh
+    - name: Upload coverage report
+      if: ${{ matrix.coverage }}
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: coverage.xml
     - name: Black
       run: |
         black --version

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.0-slim-bullseye as build
+FROM python:3.11-slim-bookworm AS build
 
 # Version of Datasette to install, e.g. 0.55
 #   docker build . -t datasette --build-arg VERSION=0.55

Justfile

@@ -49,13 +49,18 @@ export DATASETTE_SECRET := "not_a_secret"
   uv run cog -r README.md docs/*.rst
 
 # Serve live docs on localhost:8000
-@docs: cog blacken-docs
+@docs: shots cog blacken-docs
   uv run make -C docs livehtml
 
 # Build docs as static HTML
 @docs-build: cog blacken-docs
   rm -rf docs/_build && cd docs && uv run make html
 
+# Take any missing documentation screenshots defined in docs/shots.yml
+@shots:
+  uv run --group shots shot-scraper install
+  cd docs && uv run --group shots shot-scraper multi shots.yml --no-clobber --reduced-motion --retina
+
 # Apply Black
 @black:
   uv run black datasette tests

README.md

@@ -36,7 +36,7 @@ You can also install it using `pip` or `pipx`:
 
     pip install datasette
 
-Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
+Datasette requires Python 3.10 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
 
 ## Basic usage
 

datasette/__init__.py

@@ -1,6 +1,7 @@
 from datasette.permissions import Permission  # noqa
 from datasette.version import __version_info__, __version__  # noqa
 from datasette.events import Event  # noqa
+from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor  # noqa
 from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions  # noqa
 from datasette.utils.asgi import (  # noqa
     Forbidden,

datasette/_pytest_plugin.py

@@ -89,7 +89,8 @@ def pytest_runtest_protocol(item, nextitem):
                 continue
             try:
                 ds.close()
-            except Exception as e:
+            except Exception as e:  # noqa: BLE001
+                # Surfaced as a pytest warning; teardown must not fail the run
                 item.warn(
                     pytest.PytestUnraisableExceptionWarning(
                         f"Error closing Datasette instance: {e!r}"

datasette/actor_auth_cookie.py

@@ -1,8 +1,10 @@
-from datasette import hookimpl
-from itsdangerous import BadSignature
-from datasette.utils import baseconv
 import time
 
+from itsdangerous import BadSignature
+
+from datasette import hookimpl
+from datasette.utils import baseconv
+
 
 @hookimpl
 def actor_from_request(datasette, request):

datasette/background_tasks.py

@@ -0,0 +1,227 @@
+"""
+Supervised background-task registration for Datasette core.
+
+Plugins that need long-lived background work (a polling loop, a queue
+consumer, a scheduled job runner) register it with
+``datasette.add_background_task(func, name=None)`` - typically from a
+``startup`` plugin hook - instead of fire-and-forgetting their own
+``asyncio.create_task()``. Core owns:
+
+- **references**: every launched ``asyncio.Task`` is kept alive on a
+  :class:`BackgroundTaskSupervisor`, so it can never be silently garbage
+  collected the way an unreferenced ``create_task()`` call can be;
+- **launch timing**: registered work is buffered until
+  :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to
+  happen only after *every* plugin's ``startup`` hook has finished - so
+  a task that depends on another plugin having registered something first
+  doesn't need ``tryfirst=True`` ordering tricks;
+- **crash surfacing**: an unhandled exception in a background task is
+  logged with its full traceback to the ``datasette.background_tasks``
+  logger and recorded on the handle, instead of becoming an "Task
+  exception was never retrieved" warning nobody sees;
+- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels
+  every task still running and waits (with a grace period) for them to
+  actually stop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import datetime
+import functools
+import logging
+from collections.abc import Awaitable, Callable
+
+logger = logging.getLogger("datasette.background_tasks")
+
+
+def _utcnow_iso() -> str:
+    return datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+
+def _function_path(func: Callable) -> str:
+    """Describe the callable without guessing which plugin registered it."""
+    while isinstance(func, functools.partial):
+        func = func.func
+    if not hasattr(func, "__qualname__"):
+        func = type(func).__call__
+    return f"{func.__module__}.{func.__qualname__}"
+
+
+class BackgroundTask:
+    """A handle to a single piece of supervised background work.
+
+    States: ``registered`` (added but not yet launched) -> ``running`` ->
+    one of ``completed`` (returned cleanly), ``crashed`` (raised an
+    exception other than ``CancelledError`` - see ``.exception``), or
+    ``cancelled`` (``.cancel()`` was called, or it was still running at
+    shutdown).
+    """
+
+    def __init__(
+        self,
+        name: str,
+        func: Callable[[object], Awaitable[None]],
+    ):
+        self.name = name
+        self.state = "registered"
+        self.task: asyncio.Task | None = None
+        self.exception: BaseException | None = None
+        self.started_at: str | None = None
+        self.function = _function_path(func)
+        self._func = func
+        self._supervisor: BackgroundTaskSupervisor | None = None
+
+    def cancel(self) -> None:
+        """Cancel this task.
+
+        If it has already been launched, cancels the underlying
+        ``asyncio.Task`` - its state becomes ``cancelled`` once the
+        cancellation is observed (asynchronously, via the task's done
+        callback). If it has not been launched yet, this is a no-op as
+        far as asyncio is concerned (there's no task to cancel) but it
+        deregisters the handle from its supervisor so it never runs.
+        """
+        if self.task is not None:
+            self.task.cancel()
+        elif self._supervisor is not None:
+            self._supervisor._deregister(self)
+
+    def __repr__(self) -> str:
+        return f"<BackgroundTask name={self.name!r} state={self.state!r}>"
+
+
+class BackgroundTaskSupervisor:
+    """Owns registration and launch of every :class:`BackgroundTask` for a
+    single ``Datasette`` instance.
+
+    Registration (:meth:`add`) is separate from launch
+    (:meth:`launch_all`): plugins register work whenever convenient
+    (typically from a ``startup`` hook, but request handlers can register
+    dynamic per-job work too), and it either sits buffered until
+    :meth:`launch_all` runs, or - if :meth:`launch_all` has already run -
+    starts immediately.
+
+    Strong references to every :class:`BackgroundTask` (and its
+    ``asyncio.Task``) are kept for the life of the instance, by design -
+    that's what makes the enrichments-style "fire-and-forget task gets
+    garbage collected mid-flight" bug impossible here. There is currently
+    no pruning of completed/crashed/cancelled tasks, so a plugin that
+    dynamically registers many short-lived tasks over a long process
+    lifetime (a per-job registration pattern, e.g. one task per queued
+    job) will grow this list without bound. That's an accepted v1
+    trade-off in favour of full introspection (``/-/tasks``); revisit
+    with a pruning or capping policy if unbounded growth is reported in
+    practice.
+    """
+
+    def __init__(self, datasette):
+        self._datasette = datasette
+        self._tasks: list[BackgroundTask] = []
+        self._names = set()
+        self._launched = False
+        self._lock = asyncio.Lock()
+
+    def add(self, func, name=None) -> BackgroundTask:
+        base_name = name or getattr(func, "__qualname__", None) or repr(func)
+        actual_name = self._unique_name(base_name)
+        handle = BackgroundTask(actual_name, func)
+        handle._supervisor = self
+        self._tasks.append(handle)
+        self._names.add(actual_name)
+        if self._launched:
+            self._launch_one(handle)
+        return handle
+
+    def _unique_name(self, base_name: str) -> str:
+        if base_name not in self._names:
+            return base_name
+        n = 2
+        while f"{base_name}-{n}" in self._names:
+            n += 1
+        return f"{base_name}-{n}"
+
+    def _deregister(self, handle: BackgroundTask) -> None:
+        try:
+            self._tasks.remove(handle)
+        except ValueError:
+            pass
+        self._names.discard(handle.name)
+
+    def _launch_one(self, handle: BackgroundTask) -> None:
+        handle.state = "running"
+        handle.started_at = _utcnow_iso()
+        handle.task = asyncio.create_task(
+            handle._func(self._datasette), name=handle.name
+        )
+        handle.task.add_done_callback(functools.partial(_on_task_done, handle))
+
+    async def launch_all(self) -> None:
+        """Launch every currently-registered task that hasn't launched
+        yet. Idempotent and safe to call concurrently: subsequent (or
+        racing) calls are no-ops once the first has set ``self._launched``.
+        """
+        if self._launched:
+            return
+        async with self._lock:
+            if self._launched:
+                return
+            self._launched = True
+            for handle in list(self._tasks):
+                if handle.task is None:
+                    self._launch_one(handle)
+
+    async def cancel_all(self, grace: float = 5.0) -> None:
+        """Cancel every task that isn't already done, then wait up to
+        ``grace`` seconds for them to actually finish. Stragglers still
+        running after that are logged by name (but left to finish or not
+        on their own - this does not forcibly kill them, asyncio has no
+        mechanism for that).
+        """
+        handles_by_task = {
+            handle.task: handle for handle in self._tasks if handle.task is not None
+        }
+        pending = [task for task in handles_by_task if not task.done()]
+        for task in pending:
+            task.cancel()
+        if not pending:
+            return
+        _done, not_done = await asyncio.wait(pending, timeout=grace)
+        if not_done:
+            names = sorted(handles_by_task[task].name for task in not_done)
+            logger.warning(
+                "%d background task(s) did not finish within the %.1fs grace "
+                "period after cancellation: %s",
+                len(names),
+                grace,
+                ", ".join(names),
+            )
+
+    def tasks(self) -> list[BackgroundTask]:
+        """Return every registered :class:`BackgroundTask`, launched or
+        not, in registration order. Used by the ``/-/tasks`` debug
+        endpoint.
+        """
+        return list(self._tasks)
+
+    @property
+    def launched(self) -> bool:
+        """Whether :meth:`launch_all` has run yet - lets ``/-/tasks``
+        distinguish "no tasks registered" from "tasks registered but
+        nothing has armed the launch yet" without reaching for the
+        private ``_launched`` attribute.
+        """
+        return self._launched
+
+
+def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None:
+    if task.cancelled():
+        handle.state = "cancelled"
+        return
+    exc = task.exception()
+    if exc is not None:
+        handle.state = "crashed"
+        handle.exception = exc
+        logger.error("Background task %r crashed", handle.name, exc_info=exc)
+        return
+    handle.state = "completed"

datasette/blob_renderer.py

@@ -1,8 +1,9 @@
-from datasette import hookimpl
-from datasette.utils.asgi import Response, BadRequest
-from datasette.utils import to_css_class
 import hashlib
 
+from datasette import hookimpl
+from datasette.utils import to_css_class
+from datasette.utils.asgi import BadRequest, Response
+
 _BLOB_COLUMN = "_blob_column"
 _BLOB_HASH = "_blob_hash"
 

datasette/cli.py

@@ -1,43 +1,45 @@
 import asyncio
-import uvicorn
-import click
-from click import formatting
-from click.types import CompositeParamType
-from click_default_group import DefaultGroup
 import functools
 import json
 import os
 import pathlib
-from runpy import run_module
 import shutil
-from subprocess import call
 import sys
 import textwrap
 import webbrowser
+from runpy import run_module
+from subprocess import call
+
+import click
+import uvicorn
+from click import formatting
+from click.types import CompositeParamType
+from click_default_group import DefaultGroup
+
 from .app import (
-    Datasette,
     DEFAULT_SETTINGS,
     SETTINGS,
     SQLITE_LIMIT_ATTACHED,
+    Datasette,
     pm,
 )
 from .inspect import inspect_tables
 from .utils import (
+    ConnectionProblem,
     LoadExtension,
+    SpatialiteConnectionProblem,
+    SpatialiteNotFound,
     StartupError,
+    StaticMount,
+    ValueAsBooleanError,
     check_connection,
     deep_dict_update,
     find_spatialite,
-    parse_metadata,
-    ConnectionProblem,
-    SpatialiteConnectionProblem,
     initial_path_for_datasette,
     pairs_to_nested_config,
+    parse_metadata,
     temporary_docker_directory,
     value_as_boolean,
-    SpatialiteNotFound,
-    StaticMount,
-    ValueAsBooleanError,
 )
 from .utils.sqlite import sqlite3
 from .utils.testing import TestClient
@@ -75,7 +77,7 @@ class Setting(CompositeParamType):
             # Datasette 1.0, we turn bare setting names into setting.name
             # Type checking for those older settings
             default = DEFAULT_SETTINGS[name]
-            name = "settings.{}".format(name)
+            name = f"settings.{name}"
             if isinstance(default, bool):
                 try:
                     return name, "true" if value_as_boolean(value) else "false"
@@ -155,7 +157,11 @@ async def inspect_(files, sqlite_extensions):
     app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
     data = {}
     for name, database in app.databases.items():
-        tables = await database.execute_fn(lambda conn: inspect_tables(conn, {}))
+
+        def _inspect_tables(conn):
+            return inspect_tables(conn, {})
+
+        tables = await database.execute_fn(_inspect_tables)
         data[name] = {
             "hash": database.hash,
             "size": database.size,
@@ -171,7 +177,6 @@ async def inspect_(files, sqlite_extensions):
 @cli.group()
 def publish():
     """Publish specified SQLite database files to the internet along with a Datasette-powered interface and API"""
-    pass
 
 
 # Register publish plugins
@@ -496,6 +501,7 @@ def uninstall(packages, yes):
     "--internal",
     type=click.Path(),
     help="Path to a persistent Datasette internal SQLite database",
+    envvar="DATASETTE_INTERNAL",
 )
 def serve(
     files,
@@ -578,27 +584,27 @@ def serve(
         # https://github.com/simonw/datasette/issues/2389
         deep_dict_update(config_data, settings_updates)
 
-    kwargs = dict(
+    kwargs = {
-        immutables=immutable,
+        "immutables": immutable,
-        cache_headers=not reload,
+        "cache_headers": not reload,
-        cors=cors,
+        "cors": cors,
-        inspect_data=inspect_data,
+        "inspect_data": inspect_data,
-        config=config_data,
+        "config": config_data,
-        metadata=metadata_data,
+        "metadata": metadata_data,
-        sqlite_extensions=sqlite_extensions,
+        "sqlite_extensions": sqlite_extensions,
-        template_dir=template_dir,
+        "template_dir": template_dir,
-        plugins_dir=plugins_dir,
+        "plugins_dir": plugins_dir,
-        static_mounts=static,
+        "static_mounts": static,
-        settings=None,  # These are passed in config= now
+        "settings": None,  # These are passed in config= now
-        memory=memory,
+        "memory": memory,
-        secret=secret,
+        "secret": secret,
-        version_note=version_note,
+        "version_note": version_note,
-        pdb=pdb,
+        "pdb": pdb,
-        crossdb=crossdb,
+        "crossdb": crossdb,
-        nolock=nolock,
+        "nolock": nolock,
-        internal=internal,
+        "internal": internal,
-        default_deny=default_deny,
+        "default_deny": default_deny,
-    )
+    }
 
     # Separate directories from files
     directories = [f for f in files if os.path.isdir(f)]
@@ -621,9 +627,7 @@ def serve(
                 conn.close()
             else:
                 raise click.ClickException(
-                    "Invalid value for '[FILES]...': Path '{}' does not exist.".format(
+                    f"Invalid value for '[FILES]...': Path '{file}' does not exist."
-                        file
-                    )
                 )
 
     # Check for duplicate files by resolving all paths to their absolute forms
@@ -664,16 +668,6 @@ def serve(
         # Private utility mechanism for writing unit tests
         return ds
 
-    # Run async soundness checks before startup hooks, since invoke_startup
-    # now populates internal tables which requires querying each database
-    run_sync(lambda: check_databases(ds))
-
-    # Run the "startup" plugin hooks
-    try:
-        run_sync(ds.invoke_startup)
-    except StartupError as e:
-        raise click.ClickException(e.args[0])
-
     if headers and not get:
         raise click.ClickException("--headers can only be used with --get")
 
@@ -681,10 +675,23 @@ def serve(
         raise click.ClickException("--token can only be used with --get")
 
     if get:
+        # --get means we don't run Uvicorn at all
+        run_sync(lambda: check_databases(ds))
+
+        try:
+            run_sync(ds.invoke_startup)
+        except StartupError as e:
+            raise click.ClickException(e.args[0])
+
+        # --get never launches background tasks: TestClient's request below
+        # flows through the full ASGI stack, including the
+        # AsgiRunOnFirstRequest fallback, which would otherwise launch them.
+        ds._suppress_background_tasks = True
+
         client = TestClient(ds)
         request_headers = {}
         if token:
-            request_headers["Authorization"] = "Bearer {}".format(token)
+            request_headers["Authorization"] = f"Bearer {token}"
         cookies = {}
         if actor:
             cookies["ds_actor"] = client.actor_cookie(json.loads(actor))
@@ -705,30 +712,54 @@ def serve(
         sys.exit(exit_code)
         return
 
-    # Start the server
+    # check_databases, invoke_startup() and the uvicorn server all run on a
-    url = None
+    # single event loop, so that anything a plugin's "startup" hook schedules
-    if root:
+    # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is
-        ds.root_enabled = True
+    # still alive when the server starts handling requests.
-        url = "http://{}:{}{}?token={}".format(
+    async def _serve_async():
-            host, port, ds.urls.path("-/auth-token"), ds._root_token
+        # Populate internal catalog tables before invoke_startup
-        )
+        await check_databases(ds)
-        click.echo(url)
+
-    if open_browser:
+        # Run the full startup sequence (immutable-database table-count
-        if url is None:
+        # precompute + the "startup" plugin hooks) via the same entry point
-            # Figure out most convenient URL - to table, database or homepage
+        # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when
-            path = run_sync(lambda: initial_path_for_datasette(ds))
+        # uvicorn's lifespan.startup fires moments later.
-            url = f"http://{host}:{port}{path}"
+        try:
-        webbrowser.open(url)
+            await ds._startup_sequence()
-    uvicorn_kwargs = dict(
+        except StartupError as e:
-        host=host, port=port, log_level="info", lifespan="on", workers=1
+            raise click.ClickException(e.args[0])
-    )
+
-    if uds:
+        # Start the server
-        uvicorn_kwargs["uds"] = uds
+        url = None
-    if ssl_keyfile:
+        if root:
-        uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile
+            ds.root_enabled = True
-    if ssl_certfile:
+            url = "http://{}:{}{}?token={}".format(
-        uvicorn_kwargs["ssl_certfile"] = ssl_certfile
+                host, port, ds.urls.path("-/auth-token"), ds._root_token
-    uvicorn.run(ds.app(), **uvicorn_kwargs)
+            )
+            click.echo(url)
+        if open_browser:
+            if url is None:
+                # Figure out most convenient URL - to table, database or homepage
+                path = await initial_path_for_datasette(ds)
+                url = f"http://{host}:{port}{path}"
+            webbrowser.open(url)
+        uvicorn_kwargs = {
+            "host": host,
+            "port": port,
+            "log_level": "info",
+            "lifespan": "on",
+            "workers": 1,
+        }
+        if uds:
+            uvicorn_kwargs["uds"] = uds
+        if ssl_keyfile:
+            uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile
+        if ssl_certfile:
+            uvicorn_kwargs["ssl_certfile"] = ssl_certfile
+        server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs))
+        await server.serve()
+
+    asyncio.run(_serve_async())
 
 
 @cli.command()
@@ -885,7 +916,7 @@ async def check_databases(ds):
             )
         except ConnectionProblem as e:
             raise click.UsageError(
-                f"Connection to {database.path} failed check: {str(e.args[0])}"
+                f"Connection to {database.path} failed check: {e.args[0]!s}"
             )
     # If --crossdb and more than SQLITE_LIMIT_ATTACHED show warning
     if (
@@ -893,9 +924,5 @@ async def check_databases(ds):
         and len([db for db in ds.databases.values() if not db.is_memory])
         > SQLITE_LIMIT_ATTACHED
     ):
-        msg = (
+        msg = f"Warning: --crossdb only works with the first {SQLITE_LIMIT_ATTACHED} attached databases"
-            "Warning: --crossdb only works with the first {} attached databases".format(
-                SQLITE_LIMIT_ATTACHED
-            )
-        )
         click.echo(click.style(msg, bold=True, fg="yellow"), err=True)

datasette/column_types.py

@@ -64,14 +64,14 @@ class ColumnType:
         Return an HTML string to render this cell value, or None to
         fall through to the default render_cell plugin hook chain.
         """
-        return None
+        return
 
     async def validate(self, value, datasette):
         """
         Validate a value before it is written. Return None if valid,
         or a string error message if invalid.
         """
-        return None
+        return
 
     async def transform_value(self, value, datasette):
         """

datasette/csrf.py

@@ -40,12 +40,12 @@ def _origin_tuple(value):
     scheme = (parsed.scheme or "").lower()
     host = (parsed.hostname or "").lower()
     if not scheme or not host:
-        raise ValueError("missing scheme or host in {!r}".format(value))
+        raise ValueError(f"missing scheme or host in {value!r}")
     port = parsed.port  # may raise ValueError on bad ports
     if port is None:
         port = DEFAULT_PORTS.get(scheme)
     if port is None:
-        raise ValueError("unknown default port for scheme {!r}".format(scheme))
+        raise ValueError(f"unknown default port for scheme {scheme!r}")
     return scheme, host, port
 
 
@@ -125,9 +125,7 @@ class CrossOriginProtectionMiddleware:
                 return
             await self._forbid(
                 send,
-                "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format(
+                f"Sec-Fetch-Site was {sec_fetch_site!r}, expected 'same-origin' or 'none'",
-                    sec_fetch_site
-                ),
             )
             return
 
@@ -141,11 +139,11 @@ class CrossOriginProtectionMiddleware:
         request_scheme = self._request_scheme(scope)
         try:
             origin_tuple = _origin_tuple(origin)
-            expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host))
+            expected_tuple = _origin_tuple(f"{request_scheme}://{host}")
         except ValueError:
             await self._forbid(
                 send,
-                "Malformed Origin {!r} or Host {!r}".format(origin, host),
+                f"Malformed Origin {origin!r} or Host {host!r}",
             )
             return
 
@@ -155,7 +153,7 @@ class CrossOriginProtectionMiddleware:
 
         await self._forbid(
             send,
-            "Origin {!r} does not match Host {!r}".format(origin, host),
+            f"Origin {origin!r} does not match Host {host!r}",
         )
 
     def _request_scheme(self, scope):
@@ -163,7 +161,8 @@ class CrossOriginProtectionMiddleware:
             try:
                 if self.datasette.setting("force_https_urls"):
                     return "https"
-            except Exception:
+            except Exception:  # noqa: BLE001, S110
+                # Settings may not be readable this early; fall back to the ASGI scheme
                 pass
         return scope.get("scheme") or "http"
 

datasette/database.py

@@ -1,33 +1,71 @@
 import asyncio
 import atexit
-from collections import namedtuple
+import contextvars
 import inspect
 import os
-from pathlib import Path
 import queue
-import sqlite_utils
 import sys
 import tempfile
 import threading
+import time
 import uuid
+from collections import namedtuple
+from pathlib import Path
 
+import sqlite_utils
+from opentelemetry import context as otel_context_api
+from opentelemetry.trace import Status, StatusCode
+
+from .inspect import inspect_hash
+from .telemetry import (
+    callback_name,
+    linked_root_span_kwargs,
+    record_operation_duration,
+    record_query_interrupted,
+    record_write_queue_wait,
+    sql_attribute,
+    sql_operation_name,
+    tracer,
+)
+from .telemetry_registry import (
+    CALLBACK,
+    DB_NAMESPACE,
+    DB_OPERATION_NAME,
+    DB_QUERY,
+    DB_QUERY_EXECUTE,
+    DB_QUERY_TEXT,
+    DB_SYSTEM,
+    DB_WRITE_EXECUTE,
+    DB_WRITE_QUEUE_WAIT,
+    EXECUTEMANY,
+    EXECUTESCRIPT,
+    INTERRUPTED,
+    ISOLATED_CONNECTION,
+    PARAM_COUNT,
+    PARAM_SETS,
+    ROWS_RETURNED,
+    SQL_ERROR_SUPPRESSED,
+    TIME_LIMIT_MS,
+    TRANSACTION,
+    TRUNCATED,
+)
 from .tracer import trace
 from .utils import (
     call_with_supported_arguments,
     detect_fts,
     detect_primary_keys,
     detect_spatialite,
+    escape_sqlite,
     get_all_foreign_keys,
     get_outbound_foreign_keys,
     md5_not_usedforsecurity,
-    sqlite_timelimit,
     sqlite3,
-    table_columns,
+    sqlite_timelimit,
     table_column_details,
+    table_columns,
 )
 from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables
-from .utils.sqlite import sqlite_hidden_table_names
+from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names
-from .inspect import inspect_hash
 
 connections = threading.local()
 
@@ -83,6 +121,7 @@ class Database:
         self.cached_hash = None
         self.cached_size = None
         self._cached_table_counts = None
+        self._cached_derived_table_dependencies = None
         self._write_thread = None
         self._write_queue = None
         self._closed = False
@@ -91,16 +130,15 @@ class Database:
         # These are used when in non-threaded mode:
         self._read_connection = None
         self._write_connection = None
-        # This is used to track all file connections so they can be closed
+        # Track file and memory connections, including reads on worker threads,
-        self._all_file_connections = []
+        # so close() can release all of them from the calling thread.
+        self._all_connections = []
         if not is_temp_disk:
             self.mode = mode
 
     def _check_not_closed(self):
         if self._closed:
-            raise DatasetteClosedError(
+            raise DatasetteClosedError(f"Database {self.name!r} has been closed")
-                "Database {!r} has been closed".format(self.name)
-            )
 
     def _remove_pending_execute_future(self, future):
         with self._pending_execute_futures_lock:
@@ -139,15 +177,18 @@ class Database:
         if write:
             extra_kwargs["isolation_level"] = "IMMEDIATE"
         if self.memory_name:
-            uri = "file:{}?mode=memory&cache=shared".format(self.memory_name)
+            uri = f"file:{self.memory_name}?mode=memory&cache=shared"
             conn = sqlite3.connect(
                 uri, uri=True, check_same_thread=False, **extra_kwargs
             )
             if not write:
                 conn.execute("PRAGMA query_only=1")
+            self._all_connections.append(conn)
             return conn
         if self.is_memory:
-            return sqlite3.connect(":memory:", uri=True)
+            conn = sqlite3.connect(":memory:", uri=True, check_same_thread=False)
+            self._all_connections.append(conn)
+            return conn
 
         # mode=ro or immutable=1?
         if self.is_mutable:
@@ -164,7 +205,7 @@ class Database:
         conn = sqlite3.connect(
             f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs
         )
-        self._all_file_connections.append(conn)
+        self._all_connections.append(conn)
         if self.is_temp_disk and not self._wal_enabled:
             conn.execute("PRAGMA journal_mode=WAL")
             self._wal_enabled = True
@@ -192,23 +233,22 @@ class Database:
             write_thread.join(timeout=10)
             if write_thread.is_alive():
                 sys.stderr.write(
-                    "Datasette: write thread for {!r} did not exit within 10s\n".format(
+                    f"Datasette: write thread for {self.name!r} did not exit within 10s\n"
-                        self.name
-                    )
                 )
                 sys.stderr.flush()
         for future in pending_execute_futures:
             try:
                 future.result()
-            except Exception:
+            except Exception:  # noqa: BLE001, S110
+                # Shutdown teardown - a failed pending write must not block close()
                 pass
-        # Close anything still tracked in _all_file_connections
+        # Close anything still tracked in _all_connections
-        for connection in self._all_file_connections:
+        for connection in self._all_connections:
             try:
                 connection.close()
-            except Exception:
+            except Exception:  # noqa: BLE001, S110
                 pass
-        self._all_file_connections = []
+        self._all_connections = []
         # Drop per-thread cached read connections we can reach
         try:
             delattr(connections, self._thread_local_id)
@@ -218,13 +258,13 @@ class Database:
         if self._read_connection is not None:
             try:
                 self._read_connection.close()
-            except Exception:
+            except Exception:  # noqa: BLE001, S110
                 pass
             self._read_connection = None
         if self._write_connection is not None:
             try:
                 self._write_connection.close()
-            except Exception:
+            except Exception:  # noqa: BLE001, S110
                 pass
             self._write_connection = None
         if self.is_temp_disk:
@@ -246,19 +286,46 @@ class Database:
         request=None,
         return_all=False,
         returning_limit=EXECUTE_WRITE_RETURNING_LIMIT,
+        transaction=True,
+        time_limit_ms=2000,
     ):
         self._check_not_closed()
         if returning_limit < 0:
             raise ValueError("returning_limit must be >= 0")
 
-        def _inner(conn):
+        def execute_sql(conn):
             cursor = conn.execute(sql, params or [])
             return ExecuteWriteResult.from_cursor(
                 cursor, return_all=return_all, returning_limit=returning_limit
             )
 
-        with trace("sql", database=self.name, sql=sql.strip(), params=params):
+        def _inner(conn):
-            results = await self.execute_write_fn(_inner, block=block, request=request)
+            try:
+                if time_limit_ms is None:
+                    return execute_sql(conn)
+                with sqlite_timelimit(conn, time_limit_ms):
+                    return execute_sql(conn)
+            except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:
+                if e.args == ("interrupted",):
+                    raise QueryInterrupted(e, sql, params)
+                raise
+
+        with trace(  # noqa: SIM117
+            "sql", database=self.name, sql=sql.strip(), params=params
+        ):
+            with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
+                span.set_attribute(DB_SYSTEM, "sqlite")
+                span.set_attribute(DB_NAMESPACE, self.name)
+                span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
+                operation_name = sql_operation_name(sql)
+                if operation_name:
+                    span.set_attribute(DB_OPERATION_NAME, operation_name)
+                if params:
+                    span.set_attribute(PARAM_COUNT, len(params))
+                with record_operation_duration(self.name, "write"):
+                    results = await self._execute_write_fn(
+                        _inner, block=block, request=request, transaction=transaction
+                    )
         return results
 
     async def execute_write_script(self, sql, block=True, request=None):
@@ -267,10 +334,19 @@ class Database:
         def _inner(conn):
             return conn.executescript(sql)
 
-        with trace("sql", database=self.name, sql=sql.strip(), executescript=True):
+        with trace(  # noqa: SIM117
-            results = await self.execute_write_fn(
+            "sql", database=self.name, sql=sql.strip(), executescript=True
-                _inner, block=block, transaction=False, request=request
+        ):
-            )
+            # No db.operation.name, since the script can contain multiple statements
+            with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
+                span.set_attribute(DB_SYSTEM, "sqlite")
+                span.set_attribute(DB_NAMESPACE, self.name)
+                span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
+                span.set_attribute(EXECUTESCRIPT, True)
+                with record_operation_duration(self.name, "write"):
+                    results = await self._execute_write_fn(
+                        _inner, block=block, transaction=False, request=request
+                    )
         return results
 
     async def execute_write_many(self, sql, params_seq, block=True, request=None):
@@ -290,9 +366,19 @@ class Database:
         with trace(
             "sql", database=self.name, sql=sql.strip(), executemany=True
         ) as kwargs:
-            results, count = await self.execute_write_fn(
+            with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
-                _inner, block=block, request=request
+                span.set_attribute(DB_SYSTEM, "sqlite")
-            )
+                span.set_attribute(DB_NAMESPACE, self.name)
+                span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
+                span.set_attribute(EXECUTEMANY, True)
+                operation_name = sql_operation_name(sql)
+                if operation_name:
+                    span.set_attribute(DB_OPERATION_NAME, operation_name)
+                with record_operation_duration(self.name, "write"):
+                    results, count = await self._execute_write_fn(
+                        _inner, block=block, request=request
+                    )
+                span.set_attribute(PARAM_SETS, count)
             kwargs["count"] = count
         return results
 
@@ -309,31 +395,58 @@ class Database:
             finally:
                 isolated_connection.close()
                 try:
-                    self._all_file_connections.remove(isolated_connection)
+                    self._all_connections.remove(isolated_connection)
                 except ValueError:
-                    # Was probably a memory connection
+                    # May already have been cleared by close().
                     pass
 
-        if self.ds.executor is None:
+        with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
-            # non-threaded mode
+            span.set_attribute(DB_SYSTEM, "sqlite")
-            return _run()
+            span.set_attribute(DB_NAMESPACE, self.name)
-        if not write:
+            span.set_attribute(CALLBACK, callback_name(fn))
-            # Immutable database - no writes can ever occur, so there is no
+            # Immutable databases run this on the read pool, not the write queue
-            # write queue to block; run against a fresh read-only connection
+            with record_operation_duration(self.name, "write" if write else "read"):
-            return await asyncio.get_running_loop().run_in_executor(
+                if self.ds.executor is None:
-                self.ds.executor, _run
+                    # non-threaded mode
-            )
+                    return _run()
-        # Threaded mode - send to write thread
+                if not write:
-        return await self._send_to_write_thread(fn, isolated_connection=True)
+                    # Immutable database - no writes can ever occur, so there
+                    # is no write queue to block; run against a fresh
+                    # read-only connection
+                    ctx = contextvars.copy_context()
+                    return await asyncio.get_running_loop().run_in_executor(
+                        self.ds.executor, ctx.run, _run
+                    )
+                # Threaded mode - send to write thread
+                return await self._send_to_write_thread(fn, isolated_connection=True)
 
     async def analyze_sql(self, sql, params=None) -> SQLAnalysis:
         self._check_not_closed()
 
-        return await self.execute_isolated_fn(
+        def _analyze_sql(conn):
-            lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name)
+            return analyze_sql_tables(conn, sql, params, database_name=self.name)
-        )
+
+        return await self.execute_isolated_fn(_analyze_sql)
 
     async def execute_write_fn(self, fn, block=True, transaction=True, request=None):
+        """Run `fn(conn)` on the write connection, traced as a `db.query` span.
+
+        The SQL-string write methods call `_execute_write_fn()` directly to
+        avoid creating a second span.
+        """
+        self._check_not_closed()
+        # Record the name before _wrap_fn_with_hooks() wraps fn
+        name = callback_name(fn)
+        with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
+            span.set_attribute(DB_SYSTEM, "sqlite")
+            span.set_attribute(DB_NAMESPACE, self.name)
+            span.set_attribute(CALLBACK, name)
+            with record_operation_duration(self.name, "write"):
+                return await self._execute_write_fn(
+                    fn, block=block, transaction=transaction, request=request
+                )
+
+    async def _execute_write_fn(self, fn, block=True, transaction=True, request=None):
         self._check_not_closed()
         pending_events = []
 
@@ -348,9 +461,19 @@ class Database:
                 self.ds._prepare_connection(self._write_connection, self.name)
             if transaction:
                 with self._write_connection:
+                    self._write_connection.execute("BEGIN IMMEDIATE")
                     result = fn(self._write_connection)
             else:
                 result = fn(self._write_connection)
+            if not block:
+                # There is no write thread here, so the write has already
+                # finished. Hand back the same (task_id, reply_future) shape
+                # _send_to_write_thread() returns, with the future already
+                # resolved, so the block=False path below is identical in
+                # both modes.
+                reply_future = asyncio.get_running_loop().create_future()
+                reply_future.set_result(result)
+                result = (uuid.uuid4(), reply_future)
         else:
             result = await self._send_to_write_thread(
                 fn, block=block, transaction=transaction
@@ -366,7 +489,8 @@ class Database:
             async def _dispatch_events_after_write():
                 try:
                     await reply_future
-                except Exception:
+                except Exception:  # noqa: BLE001
+                    # The write failed; skip success events regardless of why
                     # if the write failed, don't emit success events
                     return
                 for event in pending_events:
@@ -419,15 +543,24 @@ class Database:
             self._write_thread = threading.Thread(
                 target=self._execute_writes, daemon=True
             )
-            self._write_thread.name = "_execute_writes for database {}".format(
+            self._write_thread.name = f"_execute_writes for database {self.name}"
-                self.name
-            )
             self._write_thread.start()
-        task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
+        task_id = uuid.uuid4()
         loop = asyncio.get_running_loop()
         reply_future = loop.create_future()
+        # Capture the OpenTelemetry context and enqueue time for the write thread
         self._write_queue.put(
-            WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction)
+            WriteTask(
+                fn,
+                task_id,
+                loop,
+                reply_future,
+                isolated_connection,
+                transaction,
+                otel_context_api.get_current(),
+                time.time_ns(),
+                block,
+            )
         )
         if block:
             return await reply_future
@@ -441,8 +574,11 @@ class Database:
         conn = None
         try:
             conn = self.connect(write=True)
+            # Threads do not inherit the caller's context, so any spans
+            # created by prepare_connection hooks here are root spans
             self.ds._prepare_connection(conn, self.name)
-        except Exception as e:
+        except Exception as e:  # noqa: BLE001
+            # Stored and re-raised to whoever queues the next write
             conn_exception = e
         while True:
             task = self._write_queue.get()
@@ -450,43 +586,105 @@ class Database:
                 if conn is not None:
                     try:
                         conn.close()
-                    except Exception:
+                    except Exception:  # noqa: BLE001, S110
+                        # Best-effort close as the write thread exits
                         pass
                 return
-            exception = None
+            # block=True: the caller awaits the result, so the write spans
-            result = None
+            # are children of the caller's span. The token must be detached
-            if conn_exception is not None:
+            # in the finally block or the context leaks into later writes.
-                exception = conn_exception
+            # block=False: the caller may finish first, so the write spans
-            elif task.isolated_connection:
+            # are root spans with a link back to the caller's span.
-                try:
+            token = None
-                    isolated_connection = self.connect(write=True)
+            write_span_kwargs = {}
-                    try:
+            if task.block:
-                        result = task.fn(isolated_connection)
+                token = otel_context_api.attach(task.otel_context)
-                    finally:
-                        isolated_connection.close()
-                        try:
-                            self._all_file_connections.remove(isolated_connection)
-                        except ValueError:
-                            # Was probably a memory connection
-                            pass
-                except Exception as e:
-                    sys.stderr.write("{}\n".format(e))
-                    sys.stderr.flush()
-                    exception = e
             else:
-                try:
+                write_span_kwargs = linked_root_span_kwargs(task.otel_context)
-                    if task.transaction:
+            try:
-                        with conn:
+                exception = None
-                            result = task.fn(conn)
+                result = None
-                    else:
+                # Span covers the time from enqueue to dequeue
-                        result = task.fn(conn)
+                dequeued_at_ns = time.time_ns()
-                except Exception as e:
+                tracer.start_span(
-                    sys.stderr.write("{}\n".format(e))
+                    DB_WRITE_QUEUE_WAIT,
-                    sys.stderr.flush()
+                    start_time=task.enqueued_at_ns,
-                    exception = e
+                    **write_span_kwargs,
-            _deliver_write_result(task, result, exception)
+                ).end(end_time=dequeued_at_ns)
+                record_write_queue_wait(self.name, dequeued_at_ns - task.enqueued_at_ns)
+                if conn_exception is not None:
+                    exception = conn_exception
+                elif task.isolated_connection:
+                    try:
+                        with tracer.start_as_current_span(
+                            DB_WRITE_EXECUTE, **write_span_kwargs
+                        ) as span:
+                            span.set_attribute(
+                                ISOLATED_CONNECTION,
+                                task.isolated_connection,
+                            )
+                            span.set_attribute(TRANSACTION, task.transaction)
+                            isolated_connection = self.connect(write=True)
+                            try:
+                                result = task.fn(isolated_connection)
+                            finally:
+                                isolated_connection.close()
+                                try:
+                                    self._all_connections.remove(isolated_connection)
+                                except ValueError:
+                                    # May already have been cleared by close().
+                                    pass
+                    except Exception as e:  # noqa: BLE001
+                        # Write thread must survive any task failure or the database wedges
+                        sys.stderr.write(f"{e}\n")
+                        sys.stderr.flush()
+                        exception = e
+                else:
+                    try:
+                        with tracer.start_as_current_span(
+                            DB_WRITE_EXECUTE, **write_span_kwargs
+                        ) as span:
+                            span.set_attribute(
+                                ISOLATED_CONNECTION,
+                                task.isolated_connection,
+                            )
+                            span.set_attribute(TRANSACTION, task.transaction)
+                            if task.transaction:
+                                with conn:
+                                    conn.execute("BEGIN IMMEDIATE")
+                                    result = task.fn(conn)
+                            else:
+                                result = task.fn(conn)
+                    except Exception as e:  # noqa: BLE001
+                        sys.stderr.write(f"{e}\n")
+                        sys.stderr.flush()
+                        exception = e
+                _deliver_write_result(task, result, exception)
+            finally:
+                if token is not None:
+                    otel_context_api.detach(token)
 
     async def execute_fn(self, fn):
+        """Run `fn(conn)` on a read connection, traced as a `db.query` span.
+
+        `execute()` calls `_execute_fn()` directly to avoid creating a second
+        span.
+        """
+        self._check_not_closed()
+
+        def fn_in_execute_span(conn):
+            # Runs on the worker thread
+            with tracer.start_as_current_span(DB_QUERY_EXECUTE):
+                return fn(conn)
+
+        with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
+            span.set_attribute(DB_SYSTEM, "sqlite")
+            span.set_attribute(DB_NAMESPACE, self.name)
+            span.set_attribute(CALLBACK, callback_name(fn))
+            with record_operation_duration(self.name, "read"):
+                return await self._execute_fn(fn_in_execute_span)
+
+    async def _execute_fn(self, fn):
         self._check_not_closed()
         if self.ds.executor is None:
             # non-threaded mode
@@ -506,7 +704,11 @@ class Database:
 
         with self._pending_execute_futures_lock:
             self._check_not_closed()
-            future = self.ds.executor.submit(in_thread)
+            # Run in a copy of the caller's context so spans created in the
+            # thread have the correct parent. This needs a fresh copy for
+            # each submit, since a Context cannot be entered concurrently.
+            ctx = contextvars.copy_context()
+            future = self.ds.executor.submit(ctx.run, in_thread)
             self._pending_execute_futures.add(future)
         future.add_done_callback(self._remove_pending_execute_future)
         return await asyncio.wrap_future(future)
@@ -523,46 +725,101 @@ class Database:
         """Executes sql against db_name in a thread"""
         self._check_not_closed()
         page_size = page_size or self.ds.page_size
+        time_limit_ms = self.ds.sql_time_limit_ms
+        # Callers that pass a shorter custom_time_limit, such as table counts
+        # and facet suggestions, expect timeouts, so they are not span errors
+        timeout_expected = bool(custom_time_limit) and custom_time_limit < time_limit_ms
+        if timeout_expected:
+            time_limit_ms = custom_time_limit
 
         def sql_operation_in_thread(conn):
-            time_limit_ms = self.ds.sql_time_limit_ms
+            # Expected timeouts and errors with log_sql_errors=False are not
-            if custom_time_limit and custom_time_limit < time_limit_ms:
+            # recorded as span errors, so exceptions are handled explicitly
-                time_limit_ms = custom_time_limit
+            with tracer.start_as_current_span(
-
+                DB_QUERY_EXECUTE,
-            with sqlite_timelimit(conn, time_limit_ms):
+                record_exception=False,
+                set_status_on_exception=False,
+            ) as execute_span:
                 try:
-                    cursor = conn.cursor()
+                    with sqlite_timelimit(conn, time_limit_ms):
-                    cursor.execute(sql, params if params is not None else {})
+                        try:
-                    max_returned_rows = self.ds.max_returned_rows
+                            cursor = conn.cursor()
-                    if max_returned_rows == page_size:
+                            cursor.execute(sql, params if params is not None else {})
-                        max_returned_rows += 1
+                            max_returned_rows = self.ds.max_returned_rows
-                    if max_returned_rows and truncate:
+                            if max_returned_rows == page_size:
-                        rows = cursor.fetchmany(max_returned_rows + 1)
+                                max_returned_rows += 1
-                        truncated = len(rows) > max_returned_rows
+                            if max_returned_rows and truncate:
-                        rows = rows[:max_returned_rows]
+                                rows = cursor.fetchmany(max_returned_rows + 1)
-                    else:
+                                truncated = len(rows) > max_returned_rows
-                        rows = cursor.fetchall()
+                                rows = rows[:max_returned_rows]
-                        truncated = False
+                            else:
-                except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:
+                                rows = cursor.fetchall()
-                    if e.args == ("interrupted",):
+                                truncated = False
-                        raise QueryInterrupted(e, sql, params)
+                        except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:
+                            if e.args == ("interrupted",):
+                                raise QueryInterrupted(e, sql, params)
+                            if log_sql_errors:
+                                sys.stderr.write(
+                                    f"ERROR: conn={conn}, sql = {sql!r}, params = {params}: {e}\n"
+                                )
+                                sys.stderr.flush()
+                            raise
+                except QueryInterrupted as e:
+                    if not timeout_expected:
+                        execute_span.record_exception(e)
+                        execute_span.set_status(Status(StatusCode.ERROR, str(e)))
+                    raise
+                except Exception as e:
                     if log_sql_errors:
-                        sys.stderr.write(
+                        execute_span.record_exception(e)
-                            "ERROR: conn={}, sql = {}, params = {}: {}\n".format(
+                        execute_span.set_status(Status(StatusCode.ERROR, str(e)))
-                                conn, repr(sql), params, e
-                            )
-                        )
-                        sys.stderr.flush()
                     raise
 
-            if truncate:
+                if truncate:
-                return Results(rows, truncated, cursor.description)
+                    return Results(rows, truncated, cursor.description)
 
-            else:
+                else:
-                return Results(rows, False, cursor.description)
+                    return Results(rows, False, cursor.description)
 
-        with trace("sql", database=self.name, sql=sql.strip(), params=params):
+        with trace(  # noqa: SIM117
-            results = await self.execute_fn(sql_operation_in_thread)
+            "sql", database=self.name, sql=sql.strip(), params=params
+        ):
+            with tracer.start_as_current_span(
+                DB_QUERY,
+                kind=DB_QUERY.kind,
+                record_exception=False,
+                set_status_on_exception=False,
+            ) as span:
+                span.set_attribute(DB_SYSTEM, "sqlite")
+                span.set_attribute(DB_NAMESPACE, self.name)
+                span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
+                span.set_attribute(TIME_LIMIT_MS, time_limit_ms)
+                operation_name = sql_operation_name(sql)
+                if operation_name:
+                    span.set_attribute(DB_OPERATION_NAME, operation_name)
+                if params:
+                    span.set_attribute(PARAM_COUNT, len(params))
+                try:
+                    with record_operation_duration(self.name, "read"):
+                        results = await self._execute_fn(sql_operation_in_thread)
+                except QueryInterrupted as e:
+                    span.set_attribute(INTERRUPTED, True)
+                    if not timeout_expected:
+                        span.set_status(Status(StatusCode.ERROR, str(e)))
+                        span.record_exception(e)
+                        record_query_interrupted(self.name)
+                    raise
+                except Exception as e:
+                    # log_sql_errors=False callers, such as facet suggestion,
+                    # expect some queries to fail
+                    if log_sql_errors:
+                        span.record_exception(e)
+                        span.set_status(Status(StatusCode.ERROR, str(e)))
+                    else:
+                        span.set_attribute(SQL_ERROR_SUPPRESSED, True)
+                    raise
+                span.set_attribute(TRUNCATED, results.truncated)
+                span.set_attribute(ROWS_RETURNED, len(results.rows))
         return results
 
     @property
@@ -603,7 +860,7 @@ class Database:
             try:
                 table_count = (
                     await self.execute(
-                        f"select count(*) from (select * from [{table}] limit {self.count_limit + 1})",
+                        f"select count(*) from (select * from {escape_sqlite(table)} limit {self.count_limit + 1})",
                         custom_time_limit=limit,
                     )
                 ).rows[0][0]
@@ -653,17 +910,32 @@ class Database:
         )
         return [r[0] for r in results.rows]
 
+    # Named functions rather than lambdas give more useful datasette.callback
+    # span attributes
+
     async def table_columns(self, table):
-        return await self.execute_fn(lambda conn: table_columns(conn, table))
+        def _table_columns(conn):
+            return table_columns(conn, table)
+
+        return await self.execute_fn(_table_columns)
 
     async def table_column_details(self, table):
-        return await self.execute_fn(lambda conn: table_column_details(conn, table))
+        def _table_column_details(conn):
+            return table_column_details(conn, table)
+
+        return await self.execute_fn(_table_column_details)
 
     async def primary_keys(self, table):
-        return await self.execute_fn(lambda conn: detect_primary_keys(conn, table))
+        def _primary_keys(conn):
+            return detect_primary_keys(conn, table)
+
+        return await self.execute_fn(_primary_keys)
 
     async def fts_table(self, table):
-        return await self.execute_fn(lambda conn: detect_fts(conn, table))
+        def _fts_table(conn):
+            return detect_fts(conn, table)
+
+        return await self.execute_fn(_fts_table)
 
     async def label_column_for_table(self, table):
         explicit_label_column = (await self.ds.table_config(self.name, table)).get(
@@ -707,9 +979,9 @@ class Database:
             column_names
             and len(column_names) == 2
             and ("id" in column_names or "pk" in column_names)
-            and not set(column_names) == {"id", "pk"}
+            and set(column_names) != {"id", "pk"}
         ):
-            return [c for c in column_names if c not in ("id", "pk")][0]
+            return next(c for c in column_names if c not in ("id", "pk"))
         # Couldn't find a label:
         return None
 
@@ -755,6 +1027,17 @@ class Database:
 
         return hidden_tables
 
+    async def derived_table_dependencies(self):
+        """Return implementation tables and the tables they derive from."""
+        schema_version = (await self.execute("PRAGMA schema_version")).first()[0]
+        if (
+            self._cached_derived_table_dependencies is None
+            or self._cached_derived_table_dependencies[0] != schema_version
+        ):
+            dependencies = await self.execute_fn(sqlite_derived_table_dependencies)
+            self._cached_derived_table_dependencies = (schema_version, dependencies)
+        return self._cached_derived_table_dependencies[1]
+
     async def view_names(self):
         results = await self.execute("select name from sqlite_master where type='view'")
         return [r[0] for r in results.rows]
@@ -850,16 +1133,28 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event):
 
 class WriteTask:
     __slots__ = (
+        "block",
+        "enqueued_at_ns",
         "fn",
-        "task_id",
-        "loop",
-        "reply_future",
         "isolated_connection",
+        "loop",
+        "otel_context",
+        "reply_future",
+        "task_id",
         "transaction",
     )
 
     def __init__(
-        self, fn, task_id, loop, reply_future, isolated_connection, transaction
+        self,
+        fn,
+        task_id,
+        loop,
+        reply_future,
+        isolated_connection,
+        transaction,
+        otel_context,
+        enqueued_at_ns,
+        block,
     ):
         self.fn = fn
         self.task_id = task_id
@@ -867,6 +1162,9 @@ class WriteTask:
         self.reply_future = reply_future
         self.isolated_connection = isolated_connection
         self.transaction = transaction
+        self.otel_context = otel_context
+        self.enqueued_at_ns = enqueued_at_ns
+        self.block = block
 
 
 def _deliver_write_result(task, result, exception):
@@ -895,7 +1193,7 @@ class QueryInterrupted(Exception):
         self.params = params
 
     def __str__(self):
-        return "QueryInterrupted: {}".format(self.e)
+        return f"QueryInterrupted: {self.e}"
 
 
 class MultipleValues(Exception):

datasette/default_actions.py

@@ -2,8 +2,8 @@ from datasette import hookimpl
 from datasette.permissions import Action
 from datasette.resources import (
     DatabaseResource,
-    TableResource,
     QueryResource,
+    TableResource,
 )
 
 

datasette/default_column_types.py

@@ -6,6 +6,17 @@ import markupsafe
 from datasette import hookimpl
 from datasette.column_types import ColumnType, SQLiteType
 
+_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE)
+
+
+def _normalize_http_url(value):
+    if not isinstance(value, str):
+        return None
+    normalized = value.strip()
+    if not _HTTP_URL_RE.fullmatch(normalized):
+        return None
+    return normalized
+
 
 class UrlColumnType(ColumnType):
     name = "url"
@@ -15,7 +26,10 @@ class UrlColumnType(ColumnType):
     async def render_cell(self, value, column, table, database, datasette, request):
         if not value or not isinstance(value, str):
             return None
-        escaped = markupsafe.escape(value.strip())
+        normalized = _normalize_http_url(value)
+        if normalized is None:
+            return markupsafe.escape(value.strip())
+        escaped = markupsafe.escape(normalized)
         return markupsafe.Markup(f'<a href="{escaped}">{escaped}</a>')
 
     async def validate(self, value, datasette):
@@ -23,7 +37,7 @@ class UrlColumnType(ColumnType):
             return None
         if not isinstance(value, str):
             return "URL must be a string"
-        if not re.match(r"^https?://\S+$", value.strip()):
+        if _normalize_http_url(value) is None:
             return "Invalid URL"
         return None
 

datasette/default_magic_parameters.py

@@ -1,8 +1,9 @@
-from datasette import hookimpl
 import datetime
 import os
 import time
 
+from datasette import hookimpl
+
 
 def header(key, request):
     key = key.replace("_", "-").encode("utf-8")

datasette/default_permissions/__init__.py

@@ -17,18 +17,29 @@ UNION/INTERSECT operations. The order of evaluation is:
 
 from __future__ import annotations
 
-# Re-export all hooks and public utilities
-from .restrictions import (
-    actor_restrictions_sql as actor_restrictions_sql,
-    restrictions_allow_action as restrictions_allow_action,
-    ActorRestrictions as ActorRestrictions,
-)
-from .root import root_user_permissions_sql as root_user_permissions_sql
 from .config import config_permissions_sql as config_permissions_sql
+from .defaults import (
+    DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS,
+)
+from .defaults import (
+    default_action_permissions_sql as default_action_permissions_sql,
+)
 from .defaults import (
     # Avoid "datasette.default_permissions" does not explicitly export attribute
     default_allow_sql_check as default_allow_sql_check,
-    default_action_permissions_sql as default_action_permissions_sql,
-    default_query_permissions_sql as default_query_permissions_sql,
-    DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS,
 )
+from .defaults import (
+    default_query_permissions_sql as default_query_permissions_sql,
+)
+from .restrictions import (
+    ActorRestrictions as ActorRestrictions,
+)
+
+# Re-export all hooks and public utilities
+from .restrictions import (
+    actor_restrictions_sql as actor_restrictions_sql,
+)
+from .restrictions import (
+    restrictions_allow_action as restrictions_allow_action,
+)
+from .root import root_user_permissions_sql as root_user_permissions_sql

datasette/default_permissions/config.py

@@ -6,7 +6,7 @@ Applies permission rules from datasette.yaml configuration.
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -55,8 +55,8 @@ class ConfigPermissionProcessor:
 
     def __init__(
         self,
-        datasette: "Datasette",
+        datasette: Datasette,
-        actor: Optional[dict],
+        actor: dict | None,
         action: str,
     ):
         self.datasette = datasette
@@ -74,8 +74,8 @@ class ConfigPermissionProcessor:
         self.restrictions = actor.get("_r", {}) if actor else {}
 
         # Pre-compute restriction info for efficiency
-        self.restricted_databases: Set[str] = set()
+        self.restricted_databases: set[str] = set()
-        self.restricted_tables: Set[Tuple[str, str]] = set()
+        self.restricted_tables: set[tuple[str, str]] = set()
 
         if self.has_restrictions:
             self.restricted_databases = {
@@ -92,16 +92,27 @@ class ConfigPermissionProcessor:
             # Tables implicitly reference their parent databases
             self.restricted_databases.update(db for db, _ in self.restricted_tables)
 
-    def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]:
+        # Resolve identity keys once per action, rather than scanning the
+        # restriction allowlist for every configured table's allow block.
+        self.restricted_table_keys = {
+            (db, self.action_obj.normalize_child(table) if self.action_obj else table)
+            for db, table in self.restricted_tables
+        }
+
+    def evaluate_allow_block(self, allow_block: Any) -> bool | None:
         """Evaluate an allow block against the current actor."""
         if allow_block is None:
             return None
+        # Values passed using ``-s permissions.* 1`` or ``0`` are parsed as
+        # integers, but should retain the CLI's boolean 1/0 behavior.
+        if isinstance(allow_block, int) and allow_block in (0, 1):
+            return bool(allow_block)
         return actor_matches_allow(self.actor, allow_block)
 
     def is_in_restriction_allowlist(
         self,
-        parent: Optional[str],
+        parent: str | None,
-        child: Optional[str],
+        child: str | None,
     ) -> bool:
         """Check if resource is allowed by actor restrictions."""
         if not self.has_restrictions:
@@ -121,8 +132,10 @@ class ConfigPermissionProcessor:
         if parent:
             table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {})
             if child:
-                table_actions = table_restrictions.get(child, [])
+                child_key = (
-                if self.action_checks.intersection(table_actions):
+                    self.action_obj.normalize_child(child) if self.action_obj else child
+                )
+                if (parent, child_key) in self.restricted_table_keys:
                     return True
             else:
                 # Parent query should proceed if any child in this database is allowlisted
@@ -143,9 +156,9 @@ class ConfigPermissionProcessor:
 
     def add_permissions_rule(
         self,
-        parent: Optional[str],
+        parent: str | None,
-        child: Optional[str],
+        child: str | None,
-        permissions_block: Optional[dict],
+        permissions_block: dict | None,
         scope_desc: str,
     ) -> None:
         """Add a rule from a permissions:{action} block."""
@@ -165,8 +178,8 @@ class ConfigPermissionProcessor:
 
     def add_allow_block_rule(
         self,
-        parent: Optional[str],
+        parent: str | None,
-        child: Optional[str],
+        child: str | None,
         allow_block: Any,
         scope_desc: str,
     ) -> None:
@@ -198,8 +211,8 @@ class ConfigPermissionProcessor:
 
     def _add_restriction_gate_denies(
         self,
-        parent: Optional[str],
+        parent: str | None,
-        child: Optional[str],
+        child: str | None,
         is_allowed: bool,
         scope_desc: str,
     ) -> None:
@@ -231,7 +244,7 @@ class ConfigPermissionProcessor:
                     if db_name == parent:
                         self.collector.add(db_name, table_name, False, reason)
 
-    def process(self) -> Optional[PermissionSQL]:
+    def process(self) -> PermissionSQL | None:
         """Process all config rules and return combined PermissionSQL."""
         self._process_root_permissions()
         self._process_databases()
@@ -421,10 +434,10 @@ class ConfigPermissionProcessor:
 
 @hookimpl(specname="permission_resources_sql")
 async def config_permissions_sql(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
     action: str,
-) -> Optional[List[PermissionSQL]]:
+) -> list[PermissionSQL] | None:
     """
     Apply permission rules from datasette.yaml configuration.
 

datasette/default_permissions/defaults.py

@@ -6,7 +6,7 @@ Provides default allow rules for standard view/execute actions.
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -29,29 +29,28 @@ DEFAULT_ALLOW_ACTIONS = frozenset(
 
 @hookimpl(specname="permission_resources_sql")
 async def default_allow_sql_check(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
     action: str,
-) -> Optional[PermissionSQL]:
+) -> PermissionSQL | None:
     """
     Enforce the default_allow_sql setting.
 
     When default_allow_sql is false (the default), execute-sql is denied
     unless explicitly allowed by config or other rules.
     """
-    if action == "execute-sql":
+    if action == "execute-sql" and not datasette.setting("default_allow_sql"):
-        if not datasette.setting("default_allow_sql"):
+        return PermissionSQL.deny(reason="default_allow_sql is false")
-            return PermissionSQL.deny(reason="default_allow_sql is false")
 
     return None
 
 
 @hookimpl(specname="permission_resources_sql")
 async def default_action_permissions_sql(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
     action: str,
-) -> Optional[PermissionSQL]:
+) -> PermissionSQL | None:
     """
     Provide default allow rules for standard view/execute actions.
 
@@ -71,10 +70,10 @@ async def default_action_permissions_sql(
 
 @hookimpl(specname="permission_resources_sql")
 async def default_query_permissions_sql(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
     action: str,
-) -> Optional[PermissionSQL]:
+) -> PermissionSQL | None:
     actor_id = actor.get("id") if isinstance(actor, dict) else None
 
     if action not in {"view-query", "update-query", "delete-query"}:

datasette/default_permissions/helpers.py

@@ -5,7 +5,7 @@ Shared helper utilities for default permission implementations.
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List, Optional, Set
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -13,7 +13,7 @@ if TYPE_CHECKING:
 from datasette.permissions import PermissionSQL
 
 
-def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]:
+def get_action_name_variants(datasette: Datasette, action: str) -> set[str]:
     """
     Get all name variants for an action (full name and abbreviation).
 
@@ -27,7 +27,7 @@ def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]:
     return variants
 
 
-def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool:
+def action_in_list(datasette: Datasette, action: str, action_list: list) -> bool:
     """Check if an action (or its abbreviation) is in a list."""
     return bool(get_action_name_variants(datasette, action).intersection(action_list))
 
@@ -36,8 +36,8 @@ def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bo
 class PermissionRow:
     """A single permission rule row."""
 
-    parent: Optional[str]
+    parent: str | None
-    child: Optional[str]
+    child: str | None
     allow: bool
     reason: str
 
@@ -46,14 +46,14 @@ class PermissionRowCollector:
     """Collects permission rows and converts them to PermissionSQL."""
 
     def __init__(self, prefix: str = "row"):
-        self.rows: List[PermissionRow] = []
+        self.rows: list[PermissionRow] = []
         self.prefix = prefix
 
     def add(
         self,
-        parent: Optional[str],
+        parent: str | None,
-        child: Optional[str],
+        child: str | None,
-        allow: Optional[bool],
+        allow: bool | None,
         reason: str,
         if_not_none: bool = False,
     ) -> None:
@@ -62,7 +62,7 @@ class PermissionRowCollector:
             return
         self.rows.append(PermissionRow(parent, child, allow, reason))
 
-    def to_permission_sql(self) -> Optional[PermissionSQL]:
+    def to_permission_sql(self) -> PermissionSQL | None:
         """Convert collected rows to a PermissionSQL object."""
         if not self.rows:
             return None

datasette/default_permissions/restrictions.py

@@ -8,7 +8,7 @@ contains allowlists of resources the actor can access.
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -23,12 +23,12 @@ from .helpers import action_in_list, get_action_name_variants
 class ActorRestrictions:
     """Parsed actor restrictions from the _r key."""
 
-    global_actions: List[str]  # _r.a - globally allowed actions
+    global_actions: list[str]  # _r.a - globally allowed actions
     database_actions: dict  # _r.d - {db_name: [actions]}
     table_actions: dict  # _r.r - {db_name: {table: [actions]}}
 
     @classmethod
-    def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]:
+    def from_actor(cls, actor: dict | None) -> ActorRestrictions | None:
         """Parse restrictions from actor dict. Returns None if no restrictions."""
         if not actor:
             return None
@@ -44,11 +44,11 @@ class ActorRestrictions:
             table_actions=restrictions.get("r", {}),
         )
 
-    def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool:
+    def is_action_globally_allowed(self, datasette: Datasette, action: str) -> bool:
         """Check if action is in the global allowlist."""
         return action_in_list(datasette, action, self.global_actions)
 
-    def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]:
+    def get_allowed_databases(self, datasette: Datasette, action: str) -> set[str]:
         """Get database names where this action is allowed."""
         allowed = set()
         for db_name, db_actions in self.database_actions.items():
@@ -57,8 +57,8 @@ class ActorRestrictions:
         return allowed
 
     def get_allowed_tables(
-        self, datasette: "Datasette", action: str
+        self, datasette: Datasette, action: str
-    ) -> Set[Tuple[str, str]]:
+    ) -> set[tuple[str, str]]:
         """Get (database, table) pairs where this action is allowed."""
         allowed = set()
         for db_name, tables in self.table_actions.items():
@@ -70,10 +70,10 @@ class ActorRestrictions:
 
 @hookimpl(specname="permission_resources_sql")
 async def actor_restrictions_sql(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
     action: str,
-) -> Optional[List[PermissionSQL]]:
+) -> list[PermissionSQL] | None:
     """
     Handle actor restriction-based permission rules.
 
@@ -140,10 +140,10 @@ async def actor_restrictions_sql(
 
 
 def restrictions_allow_action(
-    datasette: "Datasette",
+    datasette: Datasette,
     restrictions: dict,
     action: str,
-    resource: Optional[str | Tuple[str, str]],
+    resource: str | tuple[str, str] | None,
 ) -> bool:
     """
     Check if restrictions allow the requested action on the requested resource.
@@ -185,11 +185,15 @@ def restrictions_allow_action(
     # Check table/resource level
     if resource is not None and not isinstance(resource, str) and len(resource) == 2:
         database, table = resource
-        table_allowed = restrictions.get("r", {}).get(database, {}).get(table)
+        action_obj = datasette.actions.get(action)
-        if table_allowed is not None:
+        normalize = action_obj.normalize_child if action_obj else lambda name: name
-            assert isinstance(table_allowed, list)
+        for table_name, table_allowed in (
-            if to_check.intersection(table_allowed):
+            restrictions.get("r", {}).get(database, {}).items()
-                return True
+        ):
+            if normalize(table_name) == normalize(table):
+                assert isinstance(table_allowed, list)
+                if to_check.intersection(table_allowed):
+                    return True
 
     # This action is not explicitly allowed, so reject it
     return False

datasette/default_permissions/root.py

@@ -6,7 +6,7 @@ Grants full permissions to the root user when --root flag is used.
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -17,9 +17,9 @@ from datasette.permissions import PermissionSQL
 
 @hookimpl(specname="permission_resources_sql")
 async def root_user_permissions_sql(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
-) -> Optional[PermissionSQL]:
+) -> PermissionSQL | None:
     """
     Grant root user full permissions when --root flag is used.
     """

datasette/default_permissions/sqlite_statistics.py

@@ -0,0 +1,25 @@
+"""Default table-access policy for SQLite optimizer statistics."""
+
+import json
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+
+@hookimpl
+def permission_resources_sql(action):
+    if action != "view-table":
+        return None
+    return PermissionSQL(
+        sql="""
+            SELECT database_name AS parent, value AS child, 0 AS allow,
+                'SQLite statistics tables are denied by default' AS reason
+            FROM catalog_databases
+            CROSS JOIN json_each(:sqlite_statistics_names)
+        """,
+        params={
+            "sqlite_statistics_names": json.dumps(
+                ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"]
+            )
+        },
+    )

datasette/default_permissions/tokens.py

@@ -7,7 +7,7 @@ to datasette.verify_token() so all registered handlers are tried.
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -17,15 +17,13 @@ from datasette.tokens import SignedTokenHandler
 
 
 @hookimpl
-def register_token_handler(datasette: "Datasette"):
+def register_token_handler(datasette: Datasette):
     """Register the default signed token handler."""
     return SignedTokenHandler()
 
 
 @hookimpl(specname="actor_from_request")
-async def actor_from_signed_api_token(
+async def actor_from_signed_api_token(datasette: Datasette, request) -> dict | None:
-    datasette: "Datasette", request
-) -> Optional[dict]:
     """
     Authenticate requests using API tokens by delegating to all registered
     token handlers via datasette.verify_token().

datasette/default_table_actions.py

@@ -20,7 +20,7 @@ def table_actions(datasette, actor, database, table, request):
                 "label": "Alter table",
                 "description": "Change columns and primary key for this table.",
                 "attrs": {
-                    "aria-label": "Alter table {}".format(table),
+                    "aria-label": f"Alter table {table}",
                     "data-table-action": "alter-table",
                 },
             }

datasette/events.py

@@ -1,8 +1,9 @@
 from abc import ABC, abstractproperty
 from dataclasses import asdict, dataclass, field
-from datasette.hookspecs import hookimpl
 from datetime import datetime, timezone
 
+from datasette.hookspecs import hookimpl
+
 
 @dataclass
 class Event(ABC):

datasette/facets.py

@@ -1,12 +1,13 @@
 import json
 import urllib
+
 from datasette import hookimpl
 from datasette.database import QueryInterrupted
 from datasette.utils import (
+    detect_json1,
     escape_sqlite,
     path_with_added_args,
     path_with_removed_args,
-    detect_json1,
     sqlite3,
 )
 
@@ -30,7 +31,7 @@ def load_facet_configs(request, table_config):
             assert (
                 len(facet_config.values()) == 1
             ), "Metadata config dicts should be {type: config}"
-            type, facet_config = list(facet_config.items())[0]
+            type, facet_config = next(iter(facet_config.items()))
             if isinstance(facet_config, str):
                 facet_config = {"simple": facet_config}
         facet_configs.setdefault(type, []).append(
@@ -38,7 +39,7 @@ def load_facet_configs(request, table_config):
         )
     qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True)
     for key, values in qs_pairs.items():
-        if key.startswith("_facet"):
+        if key == "_facet" or key.startswith("_facet_"):
             # Figure out the facet type
             if key == "_facet":
                 type = "column"
@@ -85,7 +86,7 @@ class Facet:
         self.database = database
         # For foreign key expansion. Can be None for e.g. stored SQL queries:
         self.table = table
-        self.sql = sql or f"select * from [{table}]"
+        self.sql = sql or f"select * from {escape_sqlite(table)}"
         self.params = params or []
         self.table_config = table_config
         # row_count can be None, in which case we calculate it ourselves:
@@ -160,18 +161,13 @@ class ColumnFacet(Facet):
         for column in columns:
             if column in already_enabled:
                 continue
-            suggested_facet_sql = """
+            suggested_facet_sql = f"""
-                with limited as (select * from ({sql}) limit {suggest_consider})
+                with limited as (select * from ({self.sql}) limit {self.suggest_consider})
-                select {column} as value, count(*) as n from limited
+                select {escape_sqlite(column)} as value, count(*) as n from limited
                 where value is not null
                 group by value
-                limit {limit}
+                limit {facet_size + 1}
-            """.format(
+            """
-                column=escape_sqlite(column),
-                sql=self.sql,
-                limit=facet_size + 1,
-                suggest_consider=self.suggest_consider,
-            )
             distinct_values = None
             try:
                 distinct_values = await self.ds.execute(
@@ -267,11 +263,16 @@ class ColumnFacet(Facet):
                 for row in facet_rows:
                     column_qs = column
                     if column.startswith("_"):
-                        column_qs = "{}__exact".format(column)
+                        column_qs = f"{column}__exact"
-                    selected = (column_qs, str(row["value"])) in qs_pairs
+                    selected_args = {
+                        key: str(row["value"])
+                        for key in (column_qs, f"{column}__exact")
+                        if (key, str(row["value"])) in qs_pairs
+                    }
+                    selected = bool(selected_args)
                     if selected:
                         toggle_path = path_with_removed_args(
-                            self.request, {column_qs: str(row["value"])}
+                            self.request, selected_args
                         )
                     else:
                         toggle_path = path_with_added_args(
@@ -342,12 +343,12 @@ class ArrayFacet(Facet):
                         for v in await self.ds.execute(
                             self.database,
                             (
-                                "select {column} from ({sql}) "
+                                f"select {escape_sqlite(column)} from ({self.sql}) "
-                                "where {column} is not null "
+                                f"where {escape_sqlite(column)} is not null "
-                                "and {column} != '' "
+                                f"and {escape_sqlite(column)} != '' "
-                                "and json_array_length({column}) > 0 "
+                                f"and json_array_length({escape_sqlite(column)}) > 0 "
                                 "limit 100"
-                            ).format(column=escape_sqlite(column), sql=self.sql),
+                            ),
                             self.params,
                             truncate=False,
                             custom_time_limit=self.ds.setting(
@@ -388,14 +389,14 @@ class ArrayFacet(Facet):
             source = source_and_config["source"]
             column = config.get("column") or config["simple"]
             # https://github.com/simonw/datasette/issues/448
-            facet_sql = """
+            facet_sql = f"""
-                with inner as ({sql}),
+                with inner as ({self.sql}),
                 deduped_array_items as (
                     select
                         distinct j.value,
                         inner.*
                     from
-                        json_each([inner].{col}) j
+                        json_each([inner].{escape_sqlite(column)}) j
                         join inner
                 )
                 select
@@ -406,12 +407,8 @@ class ArrayFacet(Facet):
                 group by
                     value
                 order by
-                    count(*) desc, value limit {limit}
+                    count(*) desc, value limit {facet_size + 1}
-            """.format(
+            """
-                col=escape_sqlite(column),
-                sql=self.sql,
-                limit=facet_size + 1,
-            )
             try:
                 facet_rows_results = await self.ds.execute(
                     self.database,

datasette/filters.py

@@ -1,8 +1,12 @@
-from datasette import hookimpl
-from datasette.resources import DatabaseResource
-from datasette.views.base import DatasetteError
-from datasette.utils.asgi import BadRequest
 import json
+import math
+from typing import ClassVar
+
+from datasette import hookimpl
+from datasette.resources import DatabaseResource, TableResource
+from datasette.utils.asgi import BadRequest
+from datasette.views.base import DatasetteError
+
 from .utils import detect_json1, escape_sqlite, path_with_removed_args
 
 
@@ -48,13 +52,20 @@ def search_filters(request, database, table, datasette):
         human_descriptions = []
         extra_context = {}
 
-        # Figure out which fts_table to use
+        # Figure out which trusted fts_table to use. Query string parameters can
+        # repeat this mapping (for backwards compatibility), but must not select
+        # a different table or primary key.
         table_metadata = await datasette.table_config(database, table)
         db = datasette.get_database(database)
-        fts_table = request.args.get("_fts_table")
+        fts_table = table_metadata.get("fts_table")
-        fts_table = fts_table or table_metadata.get("fts_table")
         fts_table = fts_table or await db.fts_table(table)
-        fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid"))
+        fts_pk = table_metadata.get("fts_pk", "rowid")
+        requested_fts_table = request.args.get("_fts_table")
+        requested_fts_pk = request.args.get("_fts_pk")
+        if (requested_fts_table and requested_fts_table != fts_table) or (
+            requested_fts_pk and requested_fts_pk != fts_pk
+        ):
+            raise BadRequest("Invalid _fts_table or _fts_pk")
         search_args = {
             key: request.args[key]
             for key in request.args
@@ -72,6 +83,11 @@ def search_filters(request, database, table, datasette):
         extra_context["supports_search"] = bool(fts_table)
 
         if fts_table and search_args:
+            await datasette.ensure_permission(
+                action="view-table",
+                resource=TableResource(database=database, table=fts_table),
+                actor=request.actor,
+            )
             if "_search" in search_args:
                 # Simple ?_search=xxx
                 search = search_args["_search"]
@@ -99,9 +115,9 @@ def search_filters(request, database, table, datasette):
                             fts_table=escape_sqlite(fts_table),
                             search_col=escape_sqlite(search_col),
                             match_clause=(
-                                ":search_{}".format(i)
+                                f":search_{i}"
                                 if search_mode_raw
-                                else "escape_fts(:search_{})".format(i)
+                                else f"escape_fts(:search_{i})"
                             ),
                         )
                     )
@@ -132,13 +148,18 @@ def through_filters(request, database, table, datasette):
                 through_table = through_data["table"]
                 other_column = through_data["column"]
                 value = through_data["value"]
+                await datasette.ensure_permission(
+                    action="view-table",
+                    resource=TableResource(database=database, table=through_table),
+                    actor=request.actor,
+                )
                 db = datasette.get_database(database)
                 outgoing_foreign_keys = await db.foreign_keys_for_table(through_table)
-                try:
+                fk_to_us = next(
-                    fk_to_us = [
+                    (fk for fk in outgoing_foreign_keys if fk["other_table"] == table),
-                        fk for fk in outgoing_foreign_keys if fk["other_table"] == table
+                    None,
-                    ][0]
+                )
-                except IndexError:
+                if fk_to_us is None:
                     raise DatasetteError(
                         "Invalid _through - could not find corresponding foreign key"
                     )
@@ -182,6 +203,17 @@ class Filter:
         raise NotImplementedError
 
 
+def _coerce_numeric_filter_value(value):
+    try:
+        return int(value)
+    except ValueError:
+        try:
+            converted = float(value)
+        except ValueError:
+            return value
+        return converted if math.isfinite(converted) else value
+
+
 class TemplatedFilter(Filter):
     def __init__(
         self,
@@ -203,13 +235,17 @@ class TemplatedFilter(Filter):
 
     def where_clause(self, table, column, value, param_counter):
         converted = self.format.format(value)
-        if self.numeric and converted.isdigit():
+        if self.numeric:
-            converted = int(converted)
+            converted = _coerce_numeric_filter_value(converted)
         if self.no_argument:
-            kwargs = {"c": column}
+            kwargs = {"c": _quote_sqlite_identifier(column)}
             converted = None
         else:
-            kwargs = {"c": column, "p": f"p{param_counter}", "t": table}
+            kwargs = {
+                "c": _quote_sqlite_identifier(column),
+                "p": f"p{param_counter}",
+                "t": _quote_sqlite_identifier(table),
+            }
         return self.sql_template.format(**kwargs), converted
 
     def human_clause(self, column, value):
@@ -223,6 +259,14 @@ class TemplatedFilter(Filter):
             return template.format(c=column, v=value)
 
 
+def _quote_sqlite_identifier(identifier):
+    # Preserve the historic always-quoted SQL generated by TemplatedFilter.
+    escaped = escape_sqlite(identifier)
+    if escaped == identifier:
+        return f'"{identifier}"'
+    return escaped
+
+
 class InFilter(Filter):
     key = "in"
     display = "in"
@@ -264,56 +308,56 @@ class Filters:
             TemplatedFilter(
                 "exact",
                 "=",
-                '"{c}" = :{p}',
+                "{c} = :{p}",
                 lambda c, v: "{c} = {v}" if v.isdigit() else '{c} = "{v}"',
             ),
             TemplatedFilter(
                 "not",
                 "!=",
-                '"{c}" != :{p}',
+                "{c} != :{p}",
                 lambda c, v: "{c} != {v}" if v.isdigit() else '{c} != "{v}"',
             ),
             TemplatedFilter(
                 "contains",
                 "contains",
-                '"{c}" like :{p}',
+                "{c} like :{p}",
                 '{c} contains "{v}"',
                 format="%{}%",
             ),
             TemplatedFilter(
                 "notcontains",
                 "does not contain",
-                '"{c}" not like :{p}',
+                "{c} not like :{p}",
                 '{c} does not contain "{v}"',
                 format="%{}%",
             ),
             TemplatedFilter(
                 "endswith",
                 "ends with",
-                '"{c}" like :{p}',
+                "{c} like :{p}",
                 '{c} ends with "{v}"',
                 format="%{}",
             ),
             TemplatedFilter(
                 "startswith",
                 "starts with",
-                '"{c}" like :{p}',
+                "{c} like :{p}",
                 '{c} starts with "{v}"',
                 format="{}%",
             ),
-            TemplatedFilter("gt", ">", '"{c}" > :{p}', "{c} > {v}", numeric=True),
+            TemplatedFilter("gt", ">", "{c} > :{p}", "{c} > {v}", numeric=True),
             TemplatedFilter(
-                "gte", "\u2265", '"{c}" >= :{p}', "{c} \u2265 {v}", numeric=True
+                "gte", "\u2265", "{c} >= :{p}", "{c} \u2265 {v}", numeric=True
             ),
-            TemplatedFilter("lt", "<", '"{c}" < :{p}', "{c} < {v}", numeric=True),
+            TemplatedFilter("lt", "<", "{c} < :{p}", "{c} < {v}", numeric=True),
             TemplatedFilter(
-                "lte", "\u2264", '"{c}" <= :{p}', "{c} \u2264 {v}", numeric=True
+                "lte", "\u2264", "{c} <= :{p}", "{c} \u2264 {v}", numeric=True
             ),
-            TemplatedFilter("like", "like", '"{c}" like :{p}', '{c} like "{v}"'),
+            TemplatedFilter("like", "like", "{c} like :{p}", '{c} like "{v}"'),
             TemplatedFilter(
-                "notlike", "not like", '"{c}" not like :{p}', '{c} not like "{v}"'
+                "notlike", "not like", "{c} not like :{p}", '{c} not like "{v}"'
             ),
-            TemplatedFilter("glob", "glob", '"{c}" glob :{p}', '{c} glob "{v}"'),
+            TemplatedFilter("glob", "glob", "{c} glob :{p}", '{c} glob "{v}"'),
             InFilter(),
             NotInFilter(),
         ]
@@ -322,13 +366,13 @@ class Filters:
                 TemplatedFilter(
                     "arraycontains",
                     "array contains",
-                    """:{p} in (select value from json_each([{t}].[{c}]))""",
+                    """:{p} in (select value from json_each({t}.{c}))""",
                     '{c} contains "{v}"',
                 ),
                 TemplatedFilter(
                     "arraynotcontains",
                     "array does not contain",
-                    """:{p} not in (select value from json_each([{t}].[{c}]))""",
+                    """:{p} not in (select value from json_each({t}.{c}))""",
                     '{c} does not contain "{v}"',
                 ),
             ]
@@ -336,36 +380,34 @@ class Filters:
             else []
         )
         + [
+            TemplatedFilter("date", "date", "date({c}) = :{p}", '"{c}" is on date {v}'),
             TemplatedFilter(
-                "date", "date", 'date("{c}") = :{p}', '"{c}" is on date {v}'
+                "isnull", "is null", "{c} is null", "{c} is null", no_argument=True
-            ),
-            TemplatedFilter(
-                "isnull", "is null", '"{c}" is null', "{c} is null", no_argument=True
             ),
             TemplatedFilter(
                 "notnull",
                 "is not null",
-                '"{c}" is not null',
+                "{c} is not null",
                 "{c} is not null",
                 no_argument=True,
             ),
             TemplatedFilter(
                 "isblank",
                 "is blank",
-                '("{c}" is null or "{c}" = "")',
+                "({c} is null or {c} = '')",
                 "{c} is blank",
                 no_argument=True,
             ),
             TemplatedFilter(
                 "notblank",
                 "is not blank",
-                '("{c}" is not null and "{c}" != "")',
+                "({c} is not null and {c} != '')",
                 "{c} is not blank",
                 no_argument=True,
             ),
         ]
     )
-    _filters_by_key = {f.key: f for f in _filters}
+    _filters_by_key: ClassVar[dict[str, Filter]] = {f.key: f for f in _filters}
 
     def __init__(self, pairs):
         self.pairs = pairs

datasette/fixtures.py

@@ -1,9 +1,10 @@
-from datasette.utils.sqlite import sqlite3
-from datasette.utils import documented
 import itertools
 import random
 import string
 
+from datasette.utils import documented
+from datasette.utils.sqlite import sqlite3
+
 __all__ = [
     "EXTRA_DATABASE_SQL",
     "TABLES",
@@ -346,9 +347,7 @@ CREATE VIEW searchable_view_configured_by_metadata AS
     + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n'
     + "\n".join(
         [
-            'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format(
+            f'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'
-                a=a, b=b, c=c, content=content
-            )
             for a, b, c, content in generate_compound_rows(1001)
         ]
     )

datasette/forbidden.py

@@ -1,4 +1,5 @@
-from datasette import hookimpl, Response
+from datasette import Response, hookimpl
+
 from .utils import add_cors_headers
 
 

datasette/handle_exception.py

@@ -1,16 +1,21 @@
-from datasette import hookimpl, Response
+import traceback
+
+from markupsafe import Markup
+
+from datasette import Response, hookimpl
+
 from .utils import add_cors_headers, error_body
 from .utils.asgi import (
     Base400,
 )
 from .views.base import DatasetteError
-from markupsafe import Markup
-import traceback
 
+# Debugger imports are deliberate - they back the "pdb" setting, which drops
+# into a debugger on unhandled exceptions
 try:
-    import ipdb as pdb
+    import ipdb as pdb  # noqa: T100
 except ImportError:
-    import pdb
+    import pdb  # noqa: T100
 
 try:
     import rich
@@ -54,6 +59,10 @@ def handle_exception(datasette, request, exception):
             body = dict(info)
             body.update(error_body(plain_message or message, status))
             return Response.json(body, status=status, headers=headers)
+        if request.path.split("?")[0].endswith(".csv"):
+            return Response.text(
+                plain_message or message, status=status, headers=headers
+            )
         info.update(
             {
                 "ok": False,
@@ -69,7 +78,7 @@ def handle_exception(datasette, request, exception):
                 dict(
                     info,
                     urls=datasette.urls,
-                    menu_links=lambda: [],
+                    menu_links=list,
                 )
             ),
             status=status,

datasette/hookspecs.py

@@ -1,5 +1,4 @@
-from pluggy import HookimplMarker
+from pluggy import HookimplMarker, HookspecMarker
-from pluggy import HookspecMarker
 
 hookspec = HookspecMarker("datasette")
 hookimpl = HookimplMarker("datasette")
@@ -10,6 +9,11 @@ def startup(datasette):
     """Fires directly after Datasette first starts running"""
 
 
+@hookspec
+def shutdown(datasette):
+    """Called once when the Datasette server is shutting down"""
+
+
 @hookspec
 def asgi_wrapper(datasette):
     """Returns an ASGI middleware callable to wrap our ASGI application with"""
@@ -46,7 +50,7 @@ def extra_body_script(
 def extra_template_vars(
     template, database, table, columns, view_name, request, datasette
 ):
-    """Extra template variables to be made available to the template - can return dict or callable or awaitable"""
+    """Extra template variables to be made available to the template - can return dict, None, callable or awaitable"""
 
 
 @hookspec

datasette/inspect.py

@@ -1,13 +1,13 @@
 import hashlib
 
 from .utils import (
-    detect_spatialite,
     detect_fts,
     detect_primary_keys,
+    detect_spatialite,
     escape_sqlite,
     get_all_foreign_keys,
-    table_columns,
     sqlite3,
+    table_columns,
 )
 
 HASH_BLOCK_SIZE = 1024 * 1024
@@ -95,10 +95,10 @@ def inspect_tables(conn, database_metadata):
                 """)
         ]
 
-    for t in tables.keys():
+    for t, table_info in tables.items():
         for hidden_table in hidden_tables:
             if t == hidden_table or t.startswith(hidden_table):
-                tables[t]["hidden"] = True
+                table_info["hidden"] = True
                 continue
 
     return tables

datasette/jump.py

@@ -21,7 +21,7 @@ class JumpSQL:
         search_text: str | None = None,
         display_name: str | None = None,
         item_type: str = "menu",
-    ) -> "JumpSQL":
+    ) -> JumpSQL:
         if search_text is None:
             search_text = " ".join(
                 text for text in (label, display_name, description) if text is not None

datasette/permissions.py

@@ -1,7 +1,11 @@
+import contextvars
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Any, NamedTuple
-import contextvars
+
+_SQLITE_IDENTIFIER_CASE = str.maketrans(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
+)
 
 # Context variable to track when permission checks should be skipped
 _skip_permission_checks = contextvars.ContextVar(
@@ -49,6 +53,15 @@ class Resource(ABC):
     # Class-level metadata (subclasses must define these)
     name: str = None  # e.g., "table", "database", "model"
     parent_class: type["Resource"] | None = None  # e.g., DatabaseResource for tables
+    case_insensitive_child: bool = False
+
+    @classmethod
+    def normalize_child(cls, child: str | None) -> str | None:
+        """Return a comparison key without changing the resource's display name."""
+        if cls.case_insensitive_child and child is not None:
+            # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold.
+            return child.translate(_SQLITE_IDENTIFIER_CASE)
+        return child
 
     # Instance-level optional extra attributes
     reasons: list[str] | None = None
@@ -72,8 +85,8 @@ class Resource(ABC):
         )
 
     def __repr__(self) -> str:
-        return "{}(parent={!r}, child={!r})".format(
+        return (
-            self.__class__.__name__, self.parent, self.child
+            f"{self.__class__.__name__}(parent={self.parent!r}, child={self.child!r})"
         )
 
     @property
@@ -129,7 +142,6 @@ class Resource(ABC):
 
         Must return two columns: parent, child
         """
-        pass
 
 
 class AllowedResource(NamedTuple):
@@ -147,6 +159,11 @@ class Action:
     resource_class: type[Resource] | None = None
     also_requires: str | None = None  # Optional action name that must also be allowed
 
+    def normalize_child(self, child: str | None) -> str | None:
+        if self.resource_class is None:
+            return child
+        return self.resource_class.normalize_child(child)
+
     @property
     def takes_parent(self) -> bool:
         """

datasette/plugins.py

@@ -1,20 +1,14 @@
 import importlib
+import importlib.metadata as importlib_metadata
+import importlib.resources as importlib_resources
 import os
-import pluggy
-from pprint import pprint
 import sys
+from pprint import pprint
+
+import pluggy
+
 from . import hookspecs
 
-if sys.version_info >= (3, 9):
-    import importlib.resources as importlib_resources
-else:
-    import importlib_resources
-if sys.version_info >= (3, 10):
-    import importlib.metadata as importlib_metadata
-else:
-    import importlib_metadata
-
-
 DEFAULT_PLUGINS = (
     "datasette.publish.heroku",
     "datasette.publish.cloudrun",
@@ -24,6 +18,7 @@ DEFAULT_PLUGINS = (
     "datasette.actor_auth_cookie",
     "datasette.default_permissions",
     "datasette.default_permissions.tokens",
+    "datasette.default_permissions.sqlite_statistics",
     "datasette.default_actions",
     "datasette.default_column_types",
     "datasette.default_magic_parameters",
@@ -85,7 +80,7 @@ if DATASETTE_LOAD_PLUGINS is not None:
                     # Ensure name can be found in plugin_to_distinfo later:
                     pm._plugin_distinfo.append((mod, distribution))
         except importlib_metadata.PackageNotFoundError:
-            sys.stderr.write("Plugin {} could not be found\n".format(package_name))
+            sys.stderr.write(f"Plugin {package_name} could not be found\n")
 
 
 # Load default plugins

datasette/publish/cloudrun.py

@@ -1,15 +1,17 @@
-from datasette import hookimpl
-import click
 import json
 import os
 import re
 from subprocess import CalledProcessError, check_call, check_output
 
+import click
+
+from datasette import hookimpl
+
+from ..utils import temporary_docker_directory
 from .common import (
     add_common_publish_arguments_and_options,
     fail_if_publish_binary_not_installed,
 )
-from ..utils import temporary_docker_directory
 
 
 @hookimpl
@@ -219,7 +221,7 @@ def publish_subcommand(publish):
 
             check_call(
                 "gcloud builds submit --tag {}{}".format(
-                    image_id, " --timeout {}".format(timeout) if timeout else ""
+                    image_id, f" --timeout {timeout}" if timeout else ""
                 ),
                 shell=True,
             )
@@ -231,7 +233,7 @@ def publish_subcommand(publish):
             ("--min-instances", min_instances),
         ):
             if value is not None:
-                extra_deploy_options.append("{} {}".format(option, value))
+                extra_deploy_options.append(f"{option} {value}")
         check_call(
             "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format(
                 image_id,
@@ -258,24 +260,16 @@ def _ensure_artifact_registry(artifact_project, artifact_region, artifact_reposi
         ) from exc
 
     describe_cmd = (
-        "gcloud artifacts repositories describe {repo} --project {project} "
+        f"gcloud artifacts repositories describe {artifact_repository} --project {artifact_project} "
-        "--location {location} --quiet"
+        f"--location {artifact_region} --quiet"
-    ).format(
-        repo=artifact_repository,
-        project=artifact_project,
-        location=artifact_region,
     )
     try:
         check_call(describe_cmd, shell=True)
         return
     except CalledProcessError:
         create_cmd = (
-            "gcloud artifacts repositories create {repo} --repository-format=docker "
+            f"gcloud artifacts repositories create {artifact_repository} --repository-format=docker "
-            '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet'
+            f'--location {artifact_region} --project {artifact_project} --description "Datasette Cloud Run images" --quiet'
-        ).format(
-            repo=artifact_repository,
-            location=artifact_region,
-            project=artifact_project,
         )
         try:
             check_call(create_cmd, shell=True)

datasette/publish/common.py

@@ -1,9 +1,11 @@
-from ..utils import StaticMount
-import click
 import os
 import shutil
 import sys
 
+import click
+
+from ..utils import StaticMount
+
 
 def add_common_publish_arguments_and_options(subcommand):
     for decorator in reversed(
@@ -76,9 +78,7 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link):
     """Exit (with error message) if ``binary` isn't installed"""
     if not shutil.which(binary):
         click.secho(
-            "Publishing to {publish_target} requires {binary} to be installed and configured".format(
+            f"Publishing to {publish_target} requires {binary} to be installed and configured",
-                publish_target=publish_target, binary=binary
-            ),
             bg="red",
             fg="white",
             bold=True,

datasette/publish/heroku.py

@@ -1,19 +1,21 @@
-from contextlib import contextmanager
-from datasette import hookimpl
-import click
 import json
 import os
 import pathlib
 import shlex
 import shutil
-from subprocess import call, check_output
 import tempfile
+from contextlib import contextmanager
+from subprocess import call, check_output
+
+import click
+
+from datasette import hookimpl
+from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata
 
 from .common import (
     add_common_publish_arguments_and_options,
     fail_if_publish_binary_not_installed,
 )
-from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata
 
 
 @hookimpl
@@ -234,7 +236,7 @@ def temporary_heroku_directory(
             extras.extend(["--static", f"{mount_point}:{mount_point}"])
 
         quoted_files = " ".join(
-            ["-i {}".format(shlex.quote(file_name)) for file_name in file_names]
+            [f"-i {shlex.quote(file_name)}" for file_name in file_names]
         )
         procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format(
             quoted_files=quoted_files, extras=" ".join(extras)

datasette/renderer.py

@@ -1,12 +1,13 @@
 import json
+
 from datasette.extras import extra_names_from_request
 from datasette.utils import (
-    error_body,
-    value_as_boolean,
-    remove_infinites,
     CustomJSONEncoder,
+    error_body,
     path_from_row_pks,
+    remove_infinites,
     sqlite3,
+    value_as_boolean,
 )
 from datasette.utils.asgi import Response
 

datasette/resources.py

@@ -25,6 +25,7 @@ class TableResource(Resource):
 
     name = "table"
     parent_class = DatabaseResource
+    case_insensitive_child = True
 
     def __init__(self, database: str, table: str):
         super().__init__(parent=database, child=table)

datasette/static/column-chooser.js

@@ -1,7 +1,9 @@
+let columnChooserInstanceCounter = 0;
+
 class ColumnChooser extends HTMLElement {
   constructor() {
     super();
-    this.attachShadow({ mode: "open" });
+    this.titleId = `column-chooser-title-${++columnChooserInstanceCounter}`;
 
     // State
     this._items = [];
@@ -26,375 +28,60 @@ class ColumnChooser extends HTMLElement {
     // Bound handlers
     this._onMove = this._onMove.bind(this);
     this._onUp = this._onUp.bind(this);
+  }
 
-    this.shadowRoot.innerHTML = `
+  connectedCallback() {
-      <style>
+    if (this._modal) return;
-        :host {
+    this.innerHTML = `
-          --ink: #0f0f0f;
+      <datasette-modal><dialog aria-labelledby="${this.titleId}">
-          --paper: #eef6ff;
-          --muted: #6b6b6b;
-          --rule: #d8e6f5;
-          --accent: #1a56db;
-          --accent-light: #e8effd;
-          --card: #ffffff;
-        }
-
-        * { box-sizing: border-box; margin: 0; padding: 0; }
-
-        dialog {
-          border: none;
-          border-radius: var(--modal-border-radius, 0.75rem);
-          padding: 0;
-          margin: auto;
-          width: 100%;
-          max-width: 420px;
-          max-height: min(640px, calc(100vh - 32px));
-          box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
-          animation: slideIn var(--modal-animation-duration, 0.2s) ease-out;
-          overflow: hidden;
-          font-family: system-ui, -apple-system, sans-serif;
-          background: var(--card);
-          -webkit-user-select: none;
-          -webkit-touch-callout: none;
-          -webkit-tap-highlight-color: transparent;
-        }
-
-        dialog[open] {
-          display: flex;
-          flex-direction: column;
-          height: min(640px, calc(100vh - 32px));
-        }
-
-        dialog::backdrop {
-          background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
-          backdrop-filter: var(--modal-backdrop-blur, blur(4px));
-          -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
-          animation: fadeIn var(--modal-animation-duration, 0.2s) ease-out;
-        }
-
-        @keyframes slideIn {
-          from {
-            opacity: 0;
-            transform: translateY(-20px) scale(0.95);
-          }
-          to {
-            opacity: 1;
-            transform: translateY(0) scale(1);
-          }
-        }
-
-        @keyframes fadeIn {
-          from { opacity: 0; }
-          to { opacity: 1; }
-        }
-
-        .modal-header {
-          padding: 20px 24px 16px;
-          border-bottom: 1px solid var(--rule);
-          display: flex;
-          align-items: center;
-          justify-content: space-between;
-          flex-shrink: 0;
-        }
-
-        .modal-title {
-          font-size: 1rem;
-          font-weight: 600;
-        }
-
-        .modal-meta {
-          font-family: ui-monospace, monospace;
-          font-size: 0.7rem;
-          color: var(--muted);
-          background: var(--paper);
-          padding: 3px 9px;
-          border-radius: 20px;
-        }
-
-        .list-toolbar {
-          padding: 6px 24px;
-          border-bottom: 1px solid var(--rule);
-          display: flex;
-          gap: 12px;
-          flex-shrink: 0;
-        }
-
-        .list-toolbar button {
-          background: var(--accent-light);
-          border: 1px solid var(--rule);
-          border-radius: 4px;
-          font-family: inherit;
-          font-size: 0.75rem;
-          color: var(--accent);
-          cursor: pointer;
-          padding: 3px 10px;
-          transition: background 0.12s, color 0.12s;
-        }
-        .list-toolbar button:hover { background: var(--accent); color: white; }
-
-        .list-wrap {
-          flex: 1;
-          overflow-y: auto;
-          overflow-x: hidden;
-          position: relative;
-          overscroll-behavior: contain;
-          -webkit-overflow-scrolling: touch;
-        }
-
-        .list-wrap::before,
-        .list-wrap::after {
-          content: '';
-          position: sticky;
-          display: block;
-          left: 0; right: 0;
-          height: 20px;
-          pointer-events: none;
-          z-index: 5;
-          transition: opacity 0.2s;
-        }
-        .list-wrap::before {
-          top: 0;
-          background: linear-gradient(to bottom, rgba(255,255,255,0.9), transparent);
-        }
-        .list-wrap::after {
-          bottom: 0;
-          background: linear-gradient(to top, rgba(255,255,255,0.9), transparent);
-          margin-top: -20px;
-        }
-
-        .scroll-zone {
-          position: absolute;
-          left: 0; right: 0;
-          height: 72px;
-          pointer-events: none;
-          z-index: 10;
-        }
-        .scroll-zone-top { top: 0; }
-        .scroll-zone-bot { bottom: 0; }
-
-        .drag-list {
-          list-style: none;
-          padding: 4px 0;
-        }
-
-        .drag-item {
-          display: flex;
-          align-items: center;
-          background: white;
-          border-bottom: 1px solid var(--rule);
-          user-select: none;
-          -webkit-user-select: none;
-          -webkit-touch-callout: none;
-          position: relative;
-          transition: background 0.08s;
-        }
-
-        .drag-item:last-child { border-bottom: none; }
-
-        .drag-handle {
-          display: flex;
-          align-items: center;
-          justify-content: center;
-          width: 48px;
-          height: 48px;
-          flex-shrink: 0;
-          cursor: grab;
-          color: #c8c4bc;
-          touch-action: none;
-          transition: color 0.15s;
-        }
-
-        .drag-handle:hover { color: var(--accent); }
-        .drag-handle svg { pointer-events: none; display: block; }
-
-        .drag-item-content {
-          display: flex;
-          align-items: center;
-          flex: 1;
-          min-width: 0;
-          cursor: pointer;
-        }
-
-        .drag-item-check {
-          display: flex;
-          align-items: center;
-          width: 32px;
-          height: 48px;
-          flex-shrink: 0;
-        }
-
-        .drag-item-check input[type="checkbox"] {
-          width: 16px;
-          height: 16px;
-          accent-color: var(--accent);
-          cursor: pointer;
-        }
-
-        .drag-item-label {
-          flex: 1;
-          font-size: 0.9rem;
-          line-height: 48px;
-          padding-right: 16px;
-          white-space: nowrap;
-          overflow: hidden;
-          text-overflow: ellipsis;
-          cursor: default;
-        }
-
-        .drag-item.is-dragging {
-          opacity: 0;
-        }
-
-        .drop-indicator {
-          position: absolute;
-          left: 48px;
-          right: 0;
-          height: 2px;
-          background: var(--accent);
-          border-radius: 99px;
-          pointer-events: none;
-          z-index: 20;
-          display: none;
-        }
-        .drop-indicator.top { top: -1px; display: block; }
-        .drop-indicator.bottom { bottom: -1px; display: block; }
-
-        .drag-ghost {
-          position: fixed;
-          pointer-events: none;
-          z-index: 9999;
-          background: white;
-          border-radius: 6px;
-          box-shadow: 0 8px 32px rgba(0,0,0,0.18), 0 2px 8px rgba(0,0,0,0.1);
-          display: flex;
-          align-items: center;
-          border: 1.5px solid var(--accent-light);
-          opacity: 0.97;
-          will-change: transform;
-          font-family: system-ui, -apple-system, sans-serif;
-        }
-
-        .scroll-pulse {
-          position: absolute;
-          left: 50%;
-          transform: translateX(-50%);
-          width: 32px;
-          height: 32px;
-          border-radius: 50%;
-          background: var(--accent);
-          opacity: 0;
-          pointer-events: none;
-          z-index: 10;
-          transition: opacity 0.15s;
-        }
-        .scroll-pulse.top { top: 8px; }
-        .scroll-pulse.bot { bottom: 8px; }
-        .scroll-pulse.active {
-          opacity: 0.18;
-          animation: pulse 0.8s ease-in-out infinite;
-        }
-
-        @keyframes pulse {
-          0%, 100% { transform: translateX(-50%) scale(1); opacity: 0.18; }
-          50% { transform: translateX(-50%) scale(1.5); opacity: 0.07; }
-        }
-
-        .modal-footer {
-          padding: 14px 20px;
-          border-top: 1px solid var(--rule);
-          display: flex;
-          align-items: center;
-          gap: 10px;
-          flex-shrink: 0;
-          background: var(--paper);
-        }
-
-        .footer-info {
-          flex: 1;
-          font-family: ui-monospace, monospace;
-          font-size: 0.68rem;
-          color: var(--muted);
-        }
-
-        .btn {
-          border: none;
-          border-radius: 5px;
-          padding: 9px 20px;
-          font-size: 0.85rem;
-          font-weight: 500;
-          cursor: pointer;
-          touch-action: manipulation;
-          font-family: inherit;
-          transition: background 0.12s;
-        }
-
-        .btn-primary {
-          background: var(--accent);
-          color: white;
-        }
-        .btn-primary:hover { background: #1448c0; }
-
-        .btn-ghost {
-          background: transparent;
-          color: var(--muted);
-          border: 1px solid var(--rule);
-        }
-        .btn-ghost:hover { background: var(--rule); color: var(--ink); }
-
-        .list-wrap::-webkit-scrollbar { width: 5px; }
-        .list-wrap::-webkit-scrollbar-track { background: transparent; }
-        .list-wrap::-webkit-scrollbar-thumb { background: var(--rule); border-radius: 99px; }
-
-        input, textarea { -webkit-user-select: auto; user-select: auto; }
-      </style>
-
-      <dialog aria-labelledby="modalTitle">
           <div class="modal-header">
-            <span class="modal-title" id="modalTitle">Choose columns</span>
+            <span class="modal-title" id="${this.titleId}">Choose columns</span>
-            <span class="modal-meta" id="selectedCount"></span>
+            <span class="modal-meta"></span>
           </div>
           <div class="list-toolbar">
-            <button id="selectAllBtn">Select all</button>
+            <button class="select-all">Select all</button>
-            <button id="deselectAllBtn">Deselect all</button>
+            <button class="deselect-all">Deselect all</button>
           </div>
-          <div class="list-wrap" id="listWrap">
+          <div class="modal-body list-wrap">
-            <div class="scroll-pulse top" id="pulseTop"></div>
+            <div class="scroll-pulse top"></div>
-            <div class="scroll-pulse bot" id="pulseBot"></div>
+            <div class="scroll-pulse bot"></div>
-            <ul class="drag-list" id="dragList"></ul>
+            <ul class="drag-list"></ul>
           </div>
           <div class="modal-footer">
-            <span class="footer-info" id="footerInfo"></span>
+            <span class="footer-info"></span>
-            <button class="btn btn-ghost" id="cancelBtn">Cancel</button>
+            <button class="modal-btn modal-btn-ghost">Cancel</button>
-            <button class="btn btn-primary" id="applyBtn">Apply</button>
+            <button class="modal-btn modal-btn-primary">Apply</button>
           </div>
-      </dialog>
+      </dialog></datasette-modal>
     `;
 
     // DOM refs
-    this._dialog = this.shadowRoot.querySelector("dialog");
+    this._modal = this.querySelector("datasette-modal");
-    this._listWrap = this.shadowRoot.getElementById("listWrap");
+    this._listWrap = this.querySelector(".list-wrap");
-    this._dragList = this.shadowRoot.getElementById("dragList");
+    this._dragList = this.querySelector(".drag-list");
-    this._pulseTop = this.shadowRoot.getElementById("pulseTop");
+    this._pulseTop = this.querySelector(".scroll-pulse.top");
-    this._pulseBot = this.shadowRoot.getElementById("pulseBot");
+    this._pulseBot = this.querySelector(".scroll-pulse.bot");
-    this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn");
+    this._selectAllBtn = this.querySelector(".select-all");
-    this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn");
+    this._deselectAllBtn = this.querySelector(".deselect-all");
-    this._cancelBtn = this.shadowRoot.getElementById("cancelBtn");
+    this._cancelBtn = this.querySelector(".modal-btn-ghost");
-    this._applyBtn = this.shadowRoot.getElementById("applyBtn");
+    this._applyBtn = this.querySelector(".modal-btn-primary");
-    this._countEl = this.shadowRoot.getElementById("selectedCount");
+    this._countEl = this.querySelector(".modal-meta");
-    this._footerEl = this.shadowRoot.getElementById("footerInfo");
+    this._footerEl = this.querySelector(".footer-info");
 
     // Event listeners
     this._selectAllBtn.addEventListener("click", () => this._selectAll());
     this._deselectAllBtn.addEventListener("click", () => this._deselectAll());
-    this._cancelBtn.addEventListener("click", () => this._close());
+    this._cancelBtn.addEventListener("click", () =>
+      this._modal.requestClose("cancel"),
+    );
     this._applyBtn.addEventListener("click", () => this._apply());
-    this._dialog.addEventListener("click", (e) => {
+    this._modal.beforeClose = () => {
-      if (e.target === this._dialog) this._close();
+      this._items = this._savedItems ? [...this._savedItems] : this._items;
-    });
+      this._checked = this._savedChecked
-    this._dialog.addEventListener("cancel", (e) => {
+        ? new Set(this._savedChecked)
-      e.preventDefault();
+        : this._checked;
-      this._close();
+      return true;
-    });
+    };
   }
 
   /**
@@ -414,19 +101,11 @@ class ColumnChooser extends HTMLElement {
     this._savedChecked = new Set(this._checked);
 
     this._render();
-    this._dialog.showModal();
+    this._modal.show();
   }
 
   // ── Internal methods ──
 
-  _close() {
-    this._items = this._savedItems ? [...this._savedItems] : this._items;
-    this._checked = this._savedChecked
-      ? new Set(this._savedChecked)
-      : this._checked;
-    this._dialog.close();
-  }
-
   _selectAll() {
     this._items.forEach((col) => this._checked.add(col));
     this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => {
@@ -445,7 +124,7 @@ class ColumnChooser extends HTMLElement {
 
   _apply() {
     const selected = this._items.filter((col) => this._checked.has(col));
-    this._dialog.close();
+    this._modal.close();
     if (this._onApply) {
       this._onApply(selected);
     }
@@ -472,11 +151,13 @@ class ColumnChooser extends HTMLElement {
           <span class="drag-item-check">
             <input type="checkbox" ${this._checked.has(col) ? "checked" : ""}>
           </span>
-          <span class="drag-item-label">${col}</span>
+          <span class="drag-item-label"></span>
         </label>
         <div class="drop-indicator"></div>
       `;
 
+      li.querySelector(".drag-item-label").textContent = col;
+
       li.querySelector("input").addEventListener("change", (e) => {
         e.target.checked ? this._checked.add(col) : this._checked.delete(col);
         this._updateCounts();
@@ -509,7 +190,7 @@ class ColumnChooser extends HTMLElement {
     this._ghostOffX = e.clientX - rect.left;
     this._ghostOffY = e.clientY - rect.top;
 
-    // Build ghost inside shadow DOM
+    // Keep the drag preview inside the dialog so it stays above the backdrop.
     this._ghost = document.createElement("div");
     this._ghost.className = "drag-ghost";
     this._ghost.style.width = rect.width + "px";
@@ -518,7 +199,7 @@ class ColumnChooser extends HTMLElement {
     this._ghost.querySelector(".drop-indicator")?.remove();
     const h = this._ghost.querySelector(".drag-handle");
     if (h) h.style.color = "var(--accent)";
-    this.shadowRoot.appendChild(this._ghost);
+    this._modal.dialog.appendChild(this._ghost);
 
     srcEl.classList.add("is-dragging");
     this._positionGhost(e.clientX, e.clientY);

datasette/static/edit-tools.js

@@ -915,6 +915,7 @@ function showTableCreateDialogError(state, message) {
 
 function setTableCreateDialogSaving(state, isSaving) {
   state.isSaving = isSaving;
+  state.modal.busy = isSaving;
   state.columnList
     .querySelectorAll("input, select, button")
     .forEach(function (control) {
@@ -2043,8 +2044,7 @@ async function createTableFromDataPreview(state) {
     var tableUrl =
       responseData.table_url ||
       fallbackTableUrl(responseData.table || payload.table);
-    state.shouldRestoreFocus = false;
+    state.modal.close({ restoreFocus: false });
-    state.dialog.close();
     if (tableUrl) {
       location.href = tableUrl;
     } else {
@@ -2118,8 +2118,7 @@ async function saveTableCreateDialog(state) {
     var tableUrl =
       responseData.table_url ||
       fallbackTableUrl(responseData.table || payload.table);
-    state.shouldRestoreFocus = false;
+    state.modal.close({ restoreFocus: false });
-    state.dialog.close();
     if (tableUrl) {
       location.href = tableUrl;
     } else {
@@ -2141,18 +2140,6 @@ function confirmDiscardTableCreateChanges(state) {
   return window.confirm("Discard this new table?");
 }
 
-function closeTableCreateDialogIfConfirmed(state) {
-  if (!state || state.isSaving) {
-    return false;
-  }
-  if (!confirmDiscardTableCreateChanges(state)) {
-    return false;
-  }
-  state.shouldRestoreFocus = true;
-  state.dialog.close();
-  return true;
-}
-
 function ensureTableCreateDialog(manager) {
   if (tableCreateDialogState) {
     return tableCreateDialogState;
@@ -2161,7 +2148,8 @@ function ensureTableCreateDialog(manager) {
     return null;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.id = TABLE_CREATE_DIALOG_ID;
   dialog.className = "table-create-dialog";
   dialog.setAttribute("aria-labelledby", "table-create-title");
@@ -2171,7 +2159,7 @@ function ensureTableCreateDialog(manager) {
     </div>
     <form class="table-create-form" method="post" novalidate>
       <p class="table-create-error" id="table-create-error" role="alert" tabindex="-1" hidden></p>
-      <div class="table-create-fields">
+      <div class="modal-body table-create-fields">
         <div class="table-create-field">
           <label class="table-create-label" for="table-create-name">Table name</label>
           <input class="table-create-input table-create-table-name" id="table-create-name" type="text" name="table" required autocomplete="off">
@@ -2198,14 +2186,15 @@ function ensureTableCreateDialog(manager) {
       <div class="modal-footer">
         <a href="#" class="table-create-mode-link table-create-from-data">Create table from data</a>
         <a href="#" class="table-create-mode-link table-create-manual" hidden>Create table manually</a>
-        <button type="button" class="btn btn-ghost table-create-cancel">Cancel</button>
+        <button type="button" class="modal-btn modal-btn-ghost table-create-cancel">Cancel</button>
-        <button type="submit" class="btn btn-primary table-create-save">Create table</button>
+        <button type="submit" class="modal-btn modal-btn-primary table-create-save">Create table</button>
       </div>
     </form>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   tableCreateDialogState = {
+    modal: modal,
     dialog: dialog,
     form: dialog.querySelector(".table-create-form"),
     title: dialog.querySelector(".modal-title"),
@@ -2225,8 +2214,6 @@ function ensureTableCreateDialog(manager) {
     manualCreateLink: dialog.querySelector(".table-create-manual"),
     cancelButton: dialog.querySelector(".table-create-cancel"),
     saveButton: dialog.querySelector(".table-create-save"),
-    currentButton: null,
-    shouldRestoreFocus: true,
     isSaving: false,
     mode: "manual",
     dataPreviewRows: null,
@@ -2266,7 +2253,7 @@ function ensureTableCreateDialog(manager) {
       tableCreateDialogState.dataTextarea.focus();
       return;
     }
-    closeTableCreateDialogIfConfirmed(tableCreateDialogState);
+    modal.requestClose("cancel");
   });
 
   tableCreateDialogState.createFromDataLink.addEventListener(
@@ -2364,36 +2351,14 @@ function ensureTableCreateDialog(manager) {
     updateTableCreateDialogButtons(tableCreateDialogState);
   });
 
-  dialog.addEventListener("click", function (ev) {
+  modal.beforeClose = function (source) {
-    if (ev.target === dialog) {
+    return confirmDiscardTableCreateChanges(tableCreateDialogState);
-      closeTableCreateDialogIfConfirmed(tableCreateDialogState);
+  };
-    }
-  });
-
-  dialog.addEventListener("keydown", function (ev) {
-    if (ev.key !== "Escape") {
-      return;
-    }
-    ev.preventDefault();
-    closeTableCreateDialogIfConfirmed(tableCreateDialogState);
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    ev.preventDefault();
-    closeTableCreateDialogIfConfirmed(tableCreateDialogState);
-  });
 
   dialog.addEventListener("close", function () {
     var state = tableCreateDialogState;
     clearTableCreateDialogError(state);
     setTableCreateDialogSaving(state, false);
-    if (
-      state.shouldRestoreFocus &&
-      state.currentButton &&
-      document.contains(state.currentButton)
-    ) {
-      state.currentButton.focus();
-    }
   });
 
   return tableCreateDialogState;
@@ -2414,15 +2379,12 @@ function openTableCreateDialog(button, manager) {
     menu.open = false;
   }
   state.manager = manager;
-  state.currentButton = button;
+
-  state.shouldRestoreFocus = true;
   state.title.textContent = "Create a table in " + data.databaseName;
   clearTableCreateDialogError(state);
   resetTableCreateDialog(state);
   loadTableCreateForeignKeyTargets(state);
-  if (!state.dialog.open) {
+  state.modal.show({ returnFocusTo: button });
-    state.dialog.showModal();
-  }
   state.tableName.focus();
 }
 
@@ -2448,6 +2410,7 @@ function initTableCreateActions(manager) {
 
 function setRowDeleteDialogBusy(state, isBusy) {
   state.isBusy = isBusy;
+  state.modal.busy = isBusy;
   state.confirmButton.disabled = isBusy;
   state.cancelButton.disabled = isBusy;
   state.confirmButton.textContent = isBusy ? "Deleting..." : "Delete row";
@@ -2694,6 +2657,7 @@ function showTableAlterDialogError(state, message) {
 
 function setTableAlterDialogSaving(state, isSaving) {
   state.isSaving = isSaving;
+  state.modal.busy = isSaving;
   state.cancelButton.disabled = isSaving;
   state.addColumnButton.disabled = isSaving;
   state.backButton.disabled = isSaving;
@@ -3829,8 +3793,7 @@ async function applyTableAlterChanges(state, result) {
       result.columnTypeAssignments || [],
       tableUrl,
     );
-    state.shouldRestoreFocus = false;
+    state.modal.close({ restoreFocus: false });
-    state.dialog.close();
     if (tableAlterResultRenamesTable(result) && tableUrl) {
       window.location.href = tableUrl;
     } else {
@@ -3891,8 +3854,7 @@ async function dropTableFromAlterDialog(state) {
     if (!response.ok || (responseData && responseData.ok === false)) {
       throw rowMutationRequestError(response, responseData);
     }
-    state.shouldRestoreFocus = false;
+    state.modal.close({ restoreFocus: false });
-    state.dialog.close();
     window.location.href = tableAlterDatabaseUrl() || "/";
   } catch (error) {
     setTableAlterDialogSaving(state, false);
@@ -3928,27 +3890,6 @@ function confirmDiscardTableAlterChanges(state) {
   return window.confirm("Discard table changes?");
 }
 
-function closeTableAlterDialogIfConfirmed(state) {
-  if (!state || state.isSaving) {
-    return false;
-  }
-  if (!confirmDiscardTableAlterChanges(state)) {
-    return false;
-  }
-  state.shouldRestoreFocus = true;
-  state.dialog.close();
-  return true;
-}
-
-function closeTableAlterDialog(state) {
-  if (!state || state.isSaving) {
-    return false;
-  }
-  state.shouldRestoreFocus = true;
-  state.dialog.close();
-  return true;
-}
-
 function ensureTableAlterDialog(manager) {
   if (tableAlterDialogState) {
     return tableAlterDialogState;
@@ -3957,7 +3898,8 @@ function ensureTableAlterDialog(manager) {
     return null;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.id = TABLE_ALTER_DIALOG_ID;
   dialog.className = "table-alter-dialog";
   dialog.setAttribute("aria-labelledby", "table-alter-title");
@@ -3967,7 +3909,7 @@ function ensureTableAlterDialog(manager) {
     </div>
     <form class="table-alter-form" method="post" novalidate>
       <p class="table-alter-error" id="table-alter-error" role="alert" tabindex="-1" hidden></p>
-      <div class="table-alter-fields">
+      <div class="modal-body table-alter-fields">
         <div class="table-alter-columns">
           <div class="table-alter-column-headings" aria-hidden="true">
             <span>Column</span>
@@ -3986,18 +3928,19 @@ function ensureTableAlterDialog(manager) {
           </div>
         </details>
       </div>
-      <div class="table-alter-review" hidden></div>
+      <div class="modal-body table-alter-review" hidden></div>
       <div class="modal-footer">
-        <button type="button" class="btn btn-danger table-alter-drop" hidden>Drop table</button>
+        <button type="button" class="modal-btn modal-btn-danger table-alter-drop" hidden>Drop table</button>
-        <button type="button" class="btn btn-ghost table-alter-back" hidden>Back</button>
+        <button type="button" class="modal-btn modal-btn-ghost table-alter-back" hidden>Back</button>
-        <button type="button" class="btn btn-ghost table-alter-cancel">Cancel</button>
+        <button type="button" class="modal-btn modal-btn-ghost table-alter-cancel">Cancel</button>
-        <button type="submit" class="btn btn-primary table-alter-save">Review changes</button>
+        <button type="submit" class="modal-btn modal-btn-primary table-alter-save">Review changes</button>
       </div>
     </form>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   tableAlterDialogState = {
+    modal: modal,
     dialog: dialog,
     form: dialog.querySelector(".table-alter-form"),
     title: dialog.querySelector(".modal-title"),
@@ -4012,8 +3955,6 @@ function ensureTableAlterDialog(manager) {
     dropButton: dialog.querySelector(".table-alter-drop"),
     cancelButton: dialog.querySelector(".table-alter-cancel"),
     saveButton: dialog.querySelector(".table-alter-save"),
-    currentButton: null,
-    shouldRestoreFocus: true,
     isSaving: false,
     initialSignature: "",
     originalTableName: "",
@@ -4055,7 +3996,7 @@ function ensureTableAlterDialog(manager) {
   });
 
   tableAlterDialogState.cancelButton.addEventListener("click", function () {
-    closeTableAlterDialog(tableAlterDialogState);
+    modal.requestClose("cancel");
   });
 
   tableAlterDialogState.dropButton.addEventListener("click", function () {
@@ -4076,36 +4017,17 @@ function ensureTableAlterDialog(manager) {
     }
   });
 
-  dialog.addEventListener("click", function (ev) {
+  modal.beforeClose = function (source) {
-    if (ev.target === dialog) {
+    return (
-      closeTableAlterDialogIfConfirmed(tableAlterDialogState);
+      source === "cancel" ||
-    }
+      confirmDiscardTableAlterChanges(tableAlterDialogState)
-  });
+    );
-
+  };
-  dialog.addEventListener("keydown", function (ev) {
-    if (ev.key !== "Escape") {
-      return;
-    }
-    ev.preventDefault();
-    closeTableAlterDialogIfConfirmed(tableAlterDialogState);
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    ev.preventDefault();
-    closeTableAlterDialogIfConfirmed(tableAlterDialogState);
-  });
 
   dialog.addEventListener("close", function () {
     var state = tableAlterDialogState;
     clearTableAlterDialogError(state);
     setTableAlterDialogSaving(state, false);
-    if (
-      state.shouldRestoreFocus &&
-      state.currentButton &&
-      document.contains(state.currentButton)
-    ) {
-      state.currentButton.focus();
-    }
   });
 
   return tableAlterDialogState;
@@ -4126,8 +4048,7 @@ function openTableAlterDialog(button, manager) {
     menu.open = false;
   }
   state.manager = manager;
-  state.currentButton = button;
+
-  state.shouldRestoreFocus = true;
   state.title.textContent = "Alter table " + data.tableName;
   clearTableAlterDialogError(state);
   resetTableAlterDialog(state, data);
@@ -4137,9 +4058,7 @@ function openTableAlterDialog(button, manager) {
     tableAlterForeignKeyTargetsUrl(),
     { filterByType: false },
   );
-  if (!state.dialog.open) {
+  state.modal.show({ returnFocusTo: button });
-    state.dialog.showModal();
-  }
   var firstName = state.columnList.querySelector(".table-alter-column-name");
   if (firstName) {
     firstName.focus();
@@ -4442,7 +4361,8 @@ function ensureRowDeleteDialog(manager) {
     return null;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.id = ROW_DELETE_DIALOG_ID;
   dialog.className = "row-delete-dialog";
   dialog.setAttribute("aria-labelledby", "row-delete-title");
@@ -4454,13 +4374,14 @@ function ensureRowDeleteDialog(manager) {
     <p class="row-delete-message" id="row-delete-message">Delete row <span class="row-delete-id"></span>?</p>
     <p class="row-delete-error" role="alert" hidden></p>
     <div class="modal-footer">
-      <button type="button" class="btn btn-ghost row-delete-cancel">Cancel</button>
+      <button type="button" class="modal-btn modal-btn-ghost row-delete-cancel">Cancel</button>
-      <button type="button" class="btn btn-primary row-delete-confirm">Delete row</button>
+      <button type="button" class="modal-btn modal-btn-primary row-delete-confirm">Delete row</button>
     </div>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   rowDeleteDialogState = {
+    modal: modal,
     dialog: dialog,
     title: dialog.querySelector(".modal-title"),
     message: dialog.querySelector(".row-delete-message"),
@@ -4473,21 +4394,10 @@ function ensureRowDeleteDialog(manager) {
     currentPkPath: null,
     manager: manager,
     isBusy: false,
-    shouldRestoreFocus: true,
   };
 
   rowDeleteDialogState.cancelButton.addEventListener("click", function () {
-    if (!rowDeleteDialogState.isBusy) {
+    modal.requestClose("cancel");
-      rowDeleteDialogState.shouldRestoreFocus = true;
-      dialog.close();
-    }
-  });
-
-  dialog.addEventListener("click", function (ev) {
-    if (ev.target === dialog && !rowDeleteDialogState.isBusy) {
-      rowDeleteDialogState.shouldRestoreFocus = true;
-      dialog.close();
-    }
   });
 
   dialog.addEventListener("keydown", function (ev) {
@@ -4499,25 +4409,6 @@ function ensureRowDeleteDialog(manager) {
       if (!rowDeleteDialogState.isBusy) {
         rowDeleteDialogState.confirmButton.click();
       }
-      return;
-    }
-    if (ev.key !== "Escape") {
-      return;
-    }
-    if (rowDeleteDialogState.isBusy) {
-      ev.preventDefault();
-      return;
-    }
-    ev.preventDefault();
-    rowDeleteDialogState.shouldRestoreFocus = true;
-    dialog.close();
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    if (rowDeleteDialogState.isBusy) {
-      ev.preventDefault();
-    } else {
-      rowDeleteDialogState.shouldRestoreFocus = true;
     }
   });
 
@@ -4525,13 +4416,6 @@ function ensureRowDeleteDialog(manager) {
     var state = rowDeleteDialogState;
     clearRowDeleteDialogError(state);
     setRowDeleteDialogBusy(state, false);
-    if (
-      state.shouldRestoreFocus &&
-      state.currentButton &&
-      document.contains(state.currentButton)
-    ) {
-      state.currentButton.focus();
-    }
   });
 
   rowDeleteDialogState.confirmButton.addEventListener(
@@ -4558,8 +4442,7 @@ function ensureRowDeleteDialog(manager) {
           throw rowMutationRequestError(response, data);
         }
         if (data && data.redirect) {
-          state.shouldRestoreFocus = false;
+          state.modal.close({ restoreFocus: false });
-          state.dialog.close();
           location.href = data.redirect;
           return;
         }
@@ -4571,8 +4454,7 @@ function ensureRowDeleteDialog(manager) {
         var statusMessage = state.currentPkPath
           ? "Deleted row " + state.currentPkPath + "."
           : "Deleted row.";
-        state.shouldRestoreFocus = false;
+        state.modal.close({ restoreFocus: false });
-        state.dialog.close();
         state.currentRow.remove();
         showRowMutationStatus(state.manager, statusMessage, false);
         if (focusTarget && document.contains(focusTarget)) {
@@ -4601,11 +4483,9 @@ function openRowDeleteDialog(button, manager) {
   }
 
   state.manager = manager;
-  state.currentButton = button;
   state.currentRow = row;
   state.currentDeleteUrl = rowDeleteUrl(row);
   state.currentPkPath = rowDisplayLabel(row);
-  state.shouldRestoreFocus = true;
 
   clearRowDeleteDialogError(state);
   setRowDeleteDialogBusy(state, false);
@@ -4617,9 +4497,7 @@ function openRowDeleteDialog(button, manager) {
   );
   state.rowId.textContent = state.currentPkPath || "this row";
 
-  if (!state.dialog.open) {
+  state.modal.show({ returnFocusTo: button });
-    state.dialog.showModal();
-  }
   state.confirmButton.focus();
 }
 
@@ -5694,6 +5572,7 @@ function setRowEditDialogLoading(state, isLoading) {
 
 function setRowEditDialogSaving(state, isSaving) {
   state.isSaving = isSaving;
+  state.modal.busy = isSaving;
   updateRowEditDialogButtons(state);
 }
 
@@ -5911,18 +5790,6 @@ function confirmDiscardRowEditChanges(state) {
   return window.confirm(message);
 }
 
-function closeRowEditDialogIfConfirmed(state) {
-  if (!state || state.isSaving) {
-    return false;
-  }
-  if (!confirmDiscardRowEditChanges(state)) {
-    return false;
-  }
-  state.shouldRestoreFocus = true;
-  state.dialog.close();
-  return true;
-}
-
 function setRowInsertDialogTitle(state) {
   var insertData = tableInsertData() || {};
   var title = rowEditIsMultipleInsert(state)
@@ -6748,38 +6615,6 @@ async function insertBulkPreviewRows(state) {
   }
 }
 
-function scheduleCloseRowEditDialogIfConfirmed(state) {
-  // Fix for an issue in Safari where hitting Esc would show
-  // the confirm() prompt asking if state should be discarded
-  // but the Esc key press would then cancel that dialog too.
-  // Wait for keyup, then move the confirm() to a fresh timer tick.
-  if (!state || state.isSaving || state.isClosePending) {
-    return false;
-  }
-  if (!rowEditDialogHasChanges(state)) {
-    state.shouldRestoreFocus = true;
-    state.dialog.close();
-    return true;
-  }
-  state.isClosePending = true;
-  var closeAfterKeyup = function () {
-    if (!state.isClosePending) {
-      return;
-    }
-    state.isClosePending = false;
-    closeRowEditDialogIfConfirmed(state);
-  };
-  var onKeyup = function (ev) {
-    if (ev.key !== "Escape") {
-      return;
-    }
-    document.removeEventListener("keyup", onKeyup, true);
-    setTimeout(closeAfterKeyup, 0);
-  };
-  document.addEventListener("keyup", onKeyup, true);
-  return true;
-}
-
 function findDataRowElement(root, rowId) {
   var elements = root.querySelectorAll("[data-row]");
   for (var i = 0; i < elements.length; i += 1) {
@@ -6869,9 +6704,8 @@ async function saveRowEditDialog(state) {
     }
     var formValues = collectRowFormValues(state);
     if (state.mode === "edit" && !Object.keys(formValues).length) {
-      state.shouldRestoreFocus = true;
       hideRowMutationStatus();
-      state.dialog.close();
+      state.modal.close();
       return;
     }
     var payload =
@@ -6904,9 +6738,8 @@ async function saveRowEditDialog(state) {
         insertedRowData,
         insertData.primaryKeys || [],
       );
-      state.shouldRestoreFocus = false;
       if (!insertedRowId) {
-        state.dialog.close();
+        state.modal.close({ restoreFocus: false });
         var missingIdStatus = showRowMutationStatus(
           state.manager,
           "Inserted row. Refresh the page to see it.",
@@ -6922,7 +6755,7 @@ async function saveRowEditDialog(state) {
       try {
         insertedRow = await fetchUpdatedRowElement(state);
       } catch (_error) {
-        state.dialog.close();
+        state.modal.close({ restoreFocus: false });
         var refreshFailedStatus = showRowMutationStatus(
           state.manager,
           "Inserted row, but could not refresh the table row. Refresh the page to see it.",
@@ -6937,7 +6770,7 @@ async function saveRowEditDialog(state) {
           rowTitleLabel(insertedRow),
         );
         var addedRow = addInsertedRowToPage(insertedRow);
-        state.dialog.close();
+        state.modal.close({ restoreFocus: false });
         showRowMutationStatus(state.manager, insertedStatusMessage, false);
         if (addedRow) {
           var insertedFocusTarget =
@@ -6946,7 +6779,7 @@ async function saveRowEditDialog(state) {
           insertedFocusTarget.focus();
         }
       } else {
-        state.dialog.close();
+        state.modal.close({ restoreFocus: false });
         var filteredStatus = showRowMutationStatus(
           state.manager,
           "Inserted row. It does not match the current filters.",
@@ -6958,8 +6791,7 @@ async function saveRowEditDialog(state) {
     }
 
     if (isRowPage()) {
-      state.shouldRestoreFocus = false;
+      state.modal.close({ restoreFocus: false });
-      state.dialog.close();
       location.reload();
       return;
     }
@@ -6995,8 +6827,7 @@ async function saveRowEditDialog(state) {
       );
     }
 
-    state.shouldRestoreFocus = false;
+    state.modal.close({ restoreFocus: false });
-    state.dialog.close();
     if (focusTarget && document.contains(focusTarget)) {
       focusTarget.focus();
     }
@@ -7140,7 +6971,8 @@ function ensureRowEditDialog(manager) {
     return null;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.id = ROW_EDIT_DIALOG_ID;
   dialog.className = "row-edit-dialog";
   dialog.setAttribute("aria-labelledby", "row-edit-title");
@@ -7152,8 +6984,8 @@ function ensureRowEditDialog(manager) {
       <p class="row-edit-summary" id="row-edit-summary" hidden></p>
       <p class="row-edit-loading" role="status" aria-live="polite">Loading row...</p>
       <p class="row-edit-error" role="alert" tabindex="-1" hidden></p>
-      <div class="row-edit-fields"></div>
+      <div class="modal-body row-edit-fields"></div>
-      <div class="row-edit-bulk" hidden>
+      <div class="modal-body row-edit-bulk" hidden>
         <div class="row-edit-bulk-editor">
           <p class="row-edit-bulk-note"><label for="row-edit-bulk-textarea">Paste TSV, CSV, or JSON</label>. You can also <button type="button" class="button-as-link row-edit-bulk-open-file">open a file</button> or drop it onto this textarea</p>
           <input class="row-edit-bulk-file-input" type="file" accept=".csv,.tsv,.json,.txt,text/csv,text/tab-separated-values,application/json,text/plain" hidden>
@@ -7170,7 +7002,7 @@ function ensureRowEditDialog(manager) {
             </div>
           </div>
           <div class="row-edit-bulk-actions">
-            <button type="button" class="btn btn-ghost row-edit-copy-template"><span class="row-edit-copy-template-label-wide">Copy spreadsheet template</span><span class="row-edit-copy-template-label-narrow">Copy template</span></button>
+            <button type="button" class="modal-btn modal-btn-ghost row-edit-copy-template"><span class="row-edit-copy-template-label-wide">Copy spreadsheet template</span><span class="row-edit-copy-template-label-narrow">Copy template</span></button>
             <span class="row-edit-bulk-template-note"><span class="row-edit-bulk-template-note-wide">You can paste the template into Google Sheets or Excel.</span><span class="row-edit-bulk-template-note-narrow">Paste into Google Sheets or Excel</span></span>
           </div>
         </div>
@@ -7183,14 +7015,15 @@ function ensureRowEditDialog(manager) {
       <div class="modal-footer">
         <a href="#" class="row-edit-mode-link row-edit-bulk-insert" hidden>Insert multiple rows</a>
         <a href="#" class="row-edit-mode-link row-edit-single-insert" hidden>Insert single row</a>
-        <button type="button" class="btn btn-ghost row-edit-cancel">Cancel</button>
+        <button type="button" class="modal-btn modal-btn-ghost row-edit-cancel">Cancel</button>
-        <button type="submit" class="btn btn-primary row-edit-save" disabled>Save</button>
+        <button type="submit" class="modal-btn modal-btn-primary row-edit-save" disabled>Save</button>
       </div>
     </form>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   rowEditDialogState = {
+    modal: modal,
     dialog: dialog,
     form: dialog.querySelector(".row-edit-form"),
     title: dialog.querySelector(".modal-title"),
@@ -7221,7 +7054,6 @@ function ensureRowEditDialog(manager) {
     singleInsertLink: dialog.querySelector(".row-edit-single-insert"),
     cancelButton: dialog.querySelector(".row-edit-cancel"),
     saveButton: dialog.querySelector(".row-edit-save"),
-    currentButton: null,
     currentRow: null,
     currentRowId: null,
     currentPkPath: null,
@@ -7249,9 +7081,7 @@ function ensureRowEditDialog(manager) {
     manager: manager,
     isLoading: false,
     isSaving: false,
-    isClosePending: false,
     hasLoaded: false,
-    shouldRestoreFocus: true,
   };
 
   rowEditDialogState.form.addEventListener("submit", function (ev) {
@@ -7271,10 +7101,7 @@ function ensureRowEditDialog(manager) {
       rowEditDialogState.bulkInsertTextarea.focus();
       return;
     }
-    if (!rowEditDialogState.isSaving) {
+    modal.requestClose("cancel");
-      rowEditDialogState.shouldRestoreFocus = true;
-      dialog.close();
-    }
   });
 
   rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) {
@@ -7393,31 +7220,17 @@ function ensureRowEditDialog(manager) {
     },
   );
 
-  dialog.addEventListener("click", function (ev) {
+  modal.beforeClose = function (source) {
-    if (ev.target === dialog) {
+    return (
-      closeRowEditDialogIfConfirmed(rowEditDialogState);
+      source === "cancel" || confirmDiscardRowEditChanges(rowEditDialogState)
-    }
+    );
-  });
+  };
-
-  dialog.addEventListener("keydown", function (ev) {
-    if (ev.key !== "Escape") {
-      return;
-    }
-    ev.preventDefault();
-    scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState);
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    ev.preventDefault();
-    scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState);
-  });
 
   dialog.addEventListener("close", function () {
     var state = rowEditDialogState;
     var shouldReloadOnClose = state.shouldReloadOnClose;
     var redirectOnCloseUrl = state.redirectOnCloseUrl;
     state.loadId += 1;
-    state.isClosePending = false;
     state.bulkInsertLiveValidationError = null;
     state.shouldReloadOnClose = false;
     state.redirectOnCloseUrl = null;
@@ -7430,13 +7243,6 @@ function ensureRowEditDialog(manager) {
     destroyRowEditFields(state);
     setRowEditDialogLoading(state, false);
     setRowEditDialogSaving(state, false);
-    if (
-      state.shouldRestoreFocus &&
-      state.currentButton &&
-      document.contains(state.currentButton)
-    ) {
-      state.currentButton.focus();
-    }
     if (shouldReloadOnClose) {
       if (redirectOnCloseUrl) {
         location.href = redirectOnCloseUrl;
@@ -7461,7 +7267,6 @@ async function openRowEditDialog(button, manager) {
 
   state.manager = manager;
   state.mode = "edit";
-  state.currentButton = button;
   state.currentRow = row;
   state.currentRowId = row.getAttribute("data-row") || "";
   state.currentPkPath = rowDisplayLabel(row);
@@ -7478,7 +7283,7 @@ async function openRowEditDialog(button, manager) {
   } else {
     state.form.removeAttribute("action");
   }
-  state.shouldRestoreFocus = true;
+
   state.hasLoaded = false;
   state.loadId += 1;
   var loadId = state.loadId;
@@ -7497,9 +7302,7 @@ async function openRowEditDialog(button, manager) {
   state.summary.textContent = "";
   syncRowEditInsertModeUi(state);
 
-  if (!state.dialog.open) {
+  state.modal.show({ returnFocusTo: button });
-    state.dialog.showModal();
-  }
   state.cancelButton.focus();
 
   try {
@@ -7539,7 +7342,6 @@ function openRowInsertDialog(button, manager) {
 
   state.manager = manager;
   state.mode = "insert";
-  state.currentButton = button;
   state.currentRow = null;
   state.currentRowId = null;
   state.currentPkPath = null;
@@ -7554,7 +7356,7 @@ function openRowInsertDialog(button, manager) {
   state.shouldReloadOnClose = false;
   state.redirectOnCloseUrl = null;
   resetBulkInsertPreview(state);
-  state.shouldRestoreFocus = true;
+
   state.hasLoaded = false;
   state.loadId += 1;
 
@@ -7576,9 +7378,7 @@ function openRowInsertDialog(button, manager) {
   state.summary.textContent = "";
   syncRowEditInsertModeUi(state);
 
-  if (!state.dialog.open) {
+  state.modal.show({ returnFocusTo: button });
-    state.dialog.showModal();
-  }
   renderRowInsertFields(state, insertData);
 }
 

datasette/static/json-format-highlight-1.0.1.js

@@ -1,56 +0,0 @@
-/*
-https://github.com/luyilin/json-format-highlight
-From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js
-MIT Licensed
-*/
-(function (global, factory) {
-  typeof exports === "object" && typeof module !== "undefined"
-    ? (module.exports = factory())
-    : typeof define === "function" && define.amd
-      ? define(factory)
-      : (global.jsonFormatHighlight = factory());
-})(this, function () {
-  "use strict";
-
-  var defaultColors = {
-    keyColor: "dimgray",
-    numberColor: "lightskyblue",
-    stringColor: "lightcoral",
-    trueColor: "lightseagreen",
-    falseColor: "#f66578",
-    nullColor: "cornflowerblue",
-  };
-
-  function index(json, colorOptions) {
-    if (colorOptions === void 0) colorOptions = {};
-
-    if (!json) {
-      return;
-    }
-    if (typeof json !== "string") {
-      json = JSON.stringify(json, null, 2);
-    }
-    var colors = Object.assign({}, defaultColors, colorOptions);
-    json = json.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
-    return json.replace(
-      /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g,
-      function (match) {
-        var color = colors.numberColor;
-        if (/^"/.test(match)) {
-          color = /:$/.test(match) ? colors.keyColor : colors.stringColor;
-        } else {
-          color = /true/.test(match)
-            ? colors.trueColor
-            : /false/.test(match)
-              ? colors.falseColor
-              : /null/.test(match)
-                ? colors.nullColor
-                : color;
-        }
-        return '<span style="color: ' + color + '">' + match + "</span>";
-      },
-    );
-  }
-
-  return index;
-});

datasette/static/mobile-column-actions.js

@@ -66,7 +66,8 @@ function initMobileColumnActions(manager) {
     return;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.className = "mobile-column-actions-dialog";
   dialog.id = MOBILE_COLUMN_DIALOG_ID;
   dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID);
@@ -75,13 +76,13 @@ function initMobileColumnActions(manager) {
       <span class="modal-title" id="${MOBILE_COLUMN_DIALOG_TITLE_ID}">Column actions</span>
       <span class="modal-meta"></span>
     </div>
-    <div class="list-wrap mobile-column-list"></div>
+    <div class="modal-body list-wrap mobile-column-list"></div>
     <div class="modal-footer">
       <span class="footer-info">Tap a column to reveal actions.</span>
-      <button type="button" class="btn btn-ghost mobile-column-actions-done">Done</button>
+      <button type="button" class="modal-btn modal-btn-ghost mobile-column-actions-done">Done</button>
     </div>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   triggerButton.setAttribute("aria-haspopup", "dialog");
   triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID);
@@ -91,7 +92,6 @@ function initMobileColumnActions(manager) {
   var listWrap = dialog.querySelector(".mobile-column-list");
   var doneButton = dialog.querySelector(".mobile-column-actions-done");
   var expandedSectionId = null;
-  var shouldRestoreFocus = true;
 
   function updateExpandedSection() {
     Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => {
@@ -128,16 +128,7 @@ function initMobileColumnActions(manager) {
   }
 
   function closeDialog(options) {
-    options = options || {};
+    modal.close(options);
-    shouldRestoreFocus = options.restoreFocus !== false;
-    if (dialog.open) {
-      dialog.close();
-    } else {
-      triggerButton.setAttribute("aria-expanded", "false");
-      if (shouldRestoreFocus) {
-        triggerButton.focus();
-      }
-    }
   }
 
   function renderDialog() {
@@ -166,7 +157,8 @@ function initMobileColumnActions(manager) {
       topActions.className = "mobile-column-top-actions";
 
       var showAllColumns = document.createElement("a");
-      showAllColumns.className = "btn btn-ghost mobile-column-top-action";
+      showAllColumns.className =
+        "modal-btn modal-btn-ghost mobile-column-top-action";
       showAllColumns.href = manager.columnActions.showAllColumnsUrl();
       showAllColumns.textContent = "Show all columns";
 
@@ -265,9 +257,7 @@ function initMobileColumnActions(manager) {
     if (!renderDialog()) {
       return;
     }
-    if (!dialog.open) {
+    modal.show({ returnFocusTo: triggerButton });
-      dialog.showModal();
-    }
     triggerButton.setAttribute("aria-expanded", "true");
     var focusTarget =
       dialog.querySelector(".mobile-column-top-action") ||
@@ -288,22 +278,8 @@ function initMobileColumnActions(manager) {
     closeDialog();
   });
 
-  dialog.addEventListener("click", function (ev) {
-    if (ev.target === dialog) {
-      closeDialog();
-    }
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    ev.preventDefault();
-    closeDialog();
-  });
-
   dialog.addEventListener("close", function () {
     triggerButton.setAttribute("aria-expanded", "false");
-    if (shouldRestoreFocus) {
-      triggerButton.focus();
-    }
   });
 
   window.addEventListener("resize", function () {

datasette/static/modal.js

@@ -0,0 +1,167 @@
+// Shared lifecycle for native modal dialogs.
+(() => {
+  class DatasetteModal extends HTMLElement {
+    constructor() {
+      super();
+      this.beforeClose = null;
+      this._busy = false;
+      this._restoreFocus = true;
+      this._returnFocusTo = null;
+      this._escapeCleanup = null;
+      this._escapeTimer = null;
+    }
+
+    static create() {
+      const modal = document.createElement("datasette-modal");
+      modal.appendChild(document.createElement("dialog"));
+      return modal;
+    }
+
+    get dialog() {
+      return this.querySelector(":scope > dialog");
+    }
+
+    get busy() {
+      return this._busy;
+    }
+
+    set busy(value) {
+      this._busy = !!value;
+      if (this.dialog) {
+        this.dialog.setAttribute("aria-busy", String(this._busy));
+      }
+    }
+
+    connectedCallback() {
+      const dialog = this.dialog;
+      if (!dialog) return;
+      dialog.classList.add("datasette-modal");
+      this._listeners?.abort();
+      this._listeners = new AbortController();
+      const options = { signal: this._listeners.signal };
+      let backdropPointerDown = false;
+      const outside = (event) => {
+        const rect = dialog.getBoundingClientRect();
+        return (
+          event.target === dialog &&
+          (event.clientX < rect.left ||
+            event.clientX > rect.right ||
+            event.clientY < rect.top ||
+            event.clientY > rect.bottom)
+        );
+      };
+      dialog.addEventListener(
+        "pointerdown",
+        (event) => {
+          backdropPointerDown = outside(event);
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "click",
+        (event) => {
+          if (backdropPointerDown && outside(event))
+            this.requestClose("backdrop");
+          backdropPointerDown = false;
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "keydown",
+        (event) => {
+          if (event.key !== "Escape" || event.defaultPrevented) return;
+          // A nested native dialog or plugin picker gets first refusal.
+          if (event.target.closest("dialog") !== dialog) return;
+          event.preventDefault();
+          if (this.busy || this._escapeCleanup || this._escapeTimer !== null)
+            return;
+          // Safari can otherwise use this Escape press to cancel confirm() too.
+          // Only keyboard dismissals wait for keyup; native cancel events needn't.
+          const onKeyup = (up) => {
+            if (up.key !== "Escape") return;
+            this._escapeCleanup();
+            this._escapeCleanup = null;
+            this._escapeTimer = setTimeout(() => {
+              this._escapeTimer = null;
+              this.requestClose("escape");
+            }, 0);
+          };
+          this.ownerDocument.addEventListener("keyup", onKeyup, true);
+          this._escapeCleanup = () =>
+            this.ownerDocument.removeEventListener("keyup", onKeyup, true);
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "cancel",
+        (event) => {
+          if (event.target !== dialog) return;
+          event.preventDefault();
+          if (!this._escapeCleanup && this._escapeTimer === null)
+            this.requestClose("escape");
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "close",
+        (event) => {
+          if (event.target !== dialog || dialog.open) return;
+          this._clearPendingClose();
+          this.busy = false;
+          if (this._restoreFocus && this._returnFocusTo?.isConnected) {
+            // Menu actions may have become hidden while the dialog was open.
+            const details = this._returnFocusTo.closest("details:not([open])");
+            const target =
+              details?.querySelector("summary") || this._returnFocusTo;
+            target.focus({ preventScroll: true });
+          }
+          this._returnFocusTo = null;
+        },
+        options,
+      );
+    }
+
+    disconnectedCallback() {
+      this._listeners?.abort();
+      this._clearPendingClose();
+      this._returnFocusTo = null;
+      if (this.dialog?.open) this.dialog.close();
+      this.busy = false;
+    }
+
+    _clearPendingClose() {
+      this._escapeCleanup?.();
+      this._escapeCleanup = null;
+      clearTimeout(this._escapeTimer);
+      this._escapeTimer = null;
+    }
+
+    show({ returnFocusTo, initialFocus } = {}) {
+      const dialog = this.dialog;
+      if (!dialog.open) {
+        this._clearPendingClose();
+        this._returnFocusTo = returnFocusTo || this.ownerDocument.activeElement;
+        this._restoreFocus = true;
+        dialog.showModal();
+      }
+      if (typeof initialFocus === "function") initialFocus();
+      else initialFocus?.focus();
+    }
+
+    requestClose(source = "cancel") {
+      if (!this.dialog.open || this.busy) return false;
+      if (this.beforeClose && this.beforeClose(source) === false) return false;
+      this.close();
+      return true;
+    }
+
+    close({ restoreFocus = true } = {}) {
+      this._clearPendingClose();
+      this._restoreFocus = restoreFocus;
+      this.dialog.close();
+    }
+  }
+
+  customElements.define("datasette-modal", DatasetteModal);
+  window.DatasetteModal = DatasetteModal;
+})();

datasette/static/navigation-search.js

@@ -10,277 +10,22 @@ class NavigationSearch extends HTMLElement {
     this.recentHeadingId = `navigation-search-recent-${this.instanceId}`;
     this.statusId = `navigation-search-status-${this.instanceId}`;
     this.titleId = `navigation-search-title-${this.instanceId}`;
-    this.attachShadow({ mode: "open" });
     this.selectedIndex = -1;
     this.matches = [];
     this.renderedMatches = [];
     this.debounceTimer = null;
-    this.restoreFocusTarget = null;
+  }
-    this.shouldRestoreFocus = true;
 
+  connectedCallback() {
+    if (this._initialized) return;
+    this._initialized = true;
     this.render();
     this.setupEventListeners();
   }
 
   render() {
-    this.shadowRoot.innerHTML = `
+    this.innerHTML = `
-            <style>
+      <datasette-modal><dialog aria-modal="true" aria-labelledby="${this.titleId}">
-                :host {
-                    display: contents;
-                }
-
-                dialog {
-                    border: none;
-                    border-radius: var(--modal-border-radius, 0.75rem);
-                    padding: 0;
-                    max-width: 90vw;
-                    width: 600px;
-                    max-height: 80vh;
-                    box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
-                    animation: slideIn var(--modal-animation-duration, 0.2s) ease-out;
-                }
-
-                dialog::backdrop {
-                    background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
-                    backdrop-filter: var(--modal-backdrop-blur, blur(4px));
-                    -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
-                    animation: fadeIn var(--modal-animation-duration, 0.2s) ease-out;
-                }
-
-                @keyframes slideIn {
-                    from {
-                        opacity: 0;
-                        transform: translateY(-20px) scale(0.95);
-                    }
-                    to {
-                        opacity: 1;
-                        transform: translateY(0) scale(1);
-                    }
-                }
-
-                @keyframes fadeIn {
-                    from { opacity: 0; }
-                    to { opacity: 1; }
-                }
-
-                .search-container {
-                    display: flex;
-                    flex-direction: column;
-                }
-
-                .search-input-wrapper {
-                    padding: 1.25rem;
-                    border-bottom: 1px solid #e5e7eb;
-                    display: flex;
-                    gap: 0.5rem;
-                    align-items: center;
-                }
-
-                .search-input {
-                    width: 100%;
-                    flex: 1;
-                    min-width: 0;
-                    padding: 0.75rem 1rem;
-                    font-size: 1rem;
-                    border: 2px solid #e5e7eb;
-                    border-radius: 0.5rem;
-                    outline: none;
-                    transition: border-color 0.2s;
-                    box-sizing: border-box;
-                }
-
-                .search-input:focus {
-                    border-color: #2563eb;
-                }
-
-                .close-search {
-                    background: transparent;
-                    border: 1px solid transparent;
-                    border-radius: 0.375rem;
-                    color: #4b5563;
-                    cursor: pointer;
-                    flex: 0 0 auto;
-                    font: inherit;
-                    font-size: 1.5rem;
-                    height: 2.75rem;
-                    line-height: 1;
-                    width: 2.75rem;
-                }
-
-                .close-search:hover,
-                .close-search:focus {
-                    background-color: #f3f4f6;
-                    border-color: #d1d5db;
-                }
-
-                .results-container {
-                    overflow-y: auto;
-                    height: calc(80vh - 180px);
-                    padding: 0.5rem;
-                }
-
-                .results-list:empty {
-                    display: none;
-                }
-
-                .result-item {
-                    padding: 0.875rem 1rem;
-                    cursor: pointer;
-                    border-radius: 0.5rem;
-                    transition: background-color 0.15s;
-                    display: flex;
-                    align-items: center;
-                    gap: 0.75rem;
-                }
-
-                .result-item:hover {
-                    background-color: #f3f4f6;
-                }
-
-                .result-item.selected {
-                    background-color: #dbeafe;
-                }
-
-                .result-item > div {
-                    flex: 1;
-                    min-width: 0;
-                }
-
-                .jump-start-content {
-                    border-bottom: 1px solid #e5e7eb;
-                    margin-bottom: 0.5rem;
-                    padding: 0.5rem 0.5rem 1rem;
-                }
-
-                .jump-start-content:empty {
-                    display: none;
-                }
-
-                .result-name {
-                    font-weight: 500;
-                    color: #111827;
-                }
-
-                .result-label {
-                    font-size: 0.875rem;
-                    color: #4b5563;
-                }
-
-                .result-type {
-                    color: #4b5563;
-                    font-size: 0.75rem;
-                    font-weight: 600;
-                    text-transform: uppercase;
-                }
-
-                .result-url {
-                    font-size: 0.875rem;
-                    color: #6b7280;
-                }
-
-                .result-description {
-                    color: #374151;
-                    display: -webkit-box;
-                    font-size: 0.8125rem;
-                    line-height: 1.35;
-                    margin-top: 0.35rem;
-                    overflow: hidden;
-                    -webkit-box-orient: vertical;
-                    -webkit-line-clamp: 2;
-                }
-
-                .results-heading {
-                    color: #4b5563;
-                    font-size: 0.75rem;
-                    font-weight: 600;
-                    letter-spacing: 0;
-                    padding: 0.5rem 1rem 0.25rem;
-                    text-transform: uppercase;
-                }
-
-                .recent-actions {
-                    padding: 0.25rem 1rem 0.75rem;
-                }
-
-                .clear-recent {
-                    background: transparent;
-                    border: 0;
-                    color: #2563eb;
-                    cursor: pointer;
-                    font: inherit;
-                    font-size: 0.875rem;
-                    padding: 0;
-                }
-
-                .clear-recent:hover {
-                    text-decoration: underline;
-                }
-
-                .no-results {
-                    padding: 2rem;
-                    text-align: center;
-                    color: #6b7280;
-                }
-
-                .hint-text {
-                    padding: 0.75rem 1.25rem;
-                    font-size: 0.875rem;
-                    color: #6b7280;
-                    border-top: 1px solid #e5e7eb;
-                    display: flex;
-                    gap: 1rem;
-                    flex-wrap: wrap;
-                }
-
-                .hint-text kbd {
-                    background: #f3f4f6;
-                    padding: 0.125rem 0.375rem;
-                    border-radius: 0.25rem;
-                    font-size: 0.75rem;
-                    border: 1px solid #d1d5db;
-                    font-family: monospace;
-                }
-
-                .visually-hidden {
-                    border: 0;
-                    clip: rect(0 0 0 0);
-                    height: 1px;
-                    margin: -1px;
-                    overflow: hidden;
-                    padding: 0;
-                    position: absolute;
-                    white-space: nowrap;
-                    width: 1px;
-                }
-
-                /* Mobile optimizations */
-                @media (max-width: 640px) {
-                    dialog {
-                        width: 95vw;
-                        max-height: 85vh;
-                        border-radius: 0.5rem;
-                    }
-
-                    .search-input-wrapper {
-                        padding: 1rem;
-                    }
-
-                    .search-input {
-                        font-size: 16px; /* Prevents zoom on iOS */
-                    }
-
-                    .result-item {
-                        padding: 1rem 0.75rem;
-                    }
-
-                    .hint-text {
-                        font-size: 0.8rem;
-                        padding: 0.5rem 1rem;
-                    }
-                }
-            </style>
-
-            <dialog aria-modal="true" aria-labelledby="${this.titleId}">
                 <div class="search-container">
                     <h2 id="${this.titleId}" class="visually-hidden">Jump to</h2>
                     <p id="${this.instructionsId}" class="visually-hidden">Type to search. Use up and down arrow keys to move through results, Enter to select a result, and Escape to close this menu.</p>
@@ -302,23 +47,22 @@ class NavigationSearch extends HTMLElement {
                         >
                         <button type="button" class="close-search" aria-label="Close jump menu">&times;</button>
                     </div>
-                    <div class="results-container"></div>
+                    <div class="modal-body results-container"></div>
                     <div class="hint-text">
                         <span><kbd>↑</kbd> <kbd>↓</kbd> Navigate</span>
                         <span><kbd>Enter</kbd> Select</span>
                         <span><kbd>Esc</kbd> Close</span>
                     </div>
                 </div>
-            </dialog>
+            </dialog></datasette-modal>
         `;
   }
 
   setupEventListeners() {
-    const dialog = this.shadowRoot.querySelector("dialog");
+    const dialog = this.querySelector("dialog");
-    const input = this.shadowRoot.querySelector(".search-input");
+    const input = this.querySelector(".search-input");
-    const closeButton = this.shadowRoot.querySelector(".close-search");
+    const closeButton = this.querySelector(".close-search");
-    const resultsContainer =
+    const resultsContainer = this.querySelector(".results-container");
-      this.shadowRoot.querySelector(".results-container");
 
     // Global keyboard listener for "/"
     document.addEventListener("keydown", (e) => {
@@ -355,8 +99,6 @@ class NavigationSearch extends HTMLElement {
       } else if (e.key === "Enter") {
         e.preventDefault();
         this.selectCurrentItem();
-      } else if (e.key === "Escape") {
-        this.closeMenu();
       }
     });
 
@@ -380,18 +122,6 @@ class NavigationSearch extends HTMLElement {
       }
     });
 
-    // Close on backdrop click
-    dialog.addEventListener("click", (e) => {
-      if (e.target === dialog) {
-        this.closeMenu();
-      }
-    });
-
-    dialog.addEventListener("cancel", (e) => {
-      e.preventDefault();
-      this.closeMenu();
-    });
-
     dialog.addEventListener("close", () => {
       this.onMenuClosed();
     });
@@ -432,19 +162,6 @@ class NavigationSearch extends HTMLElement {
     }
   }
 
-  focusRestoreTarget(trigger) {
-    if (trigger && typeof trigger.focus === "function") {
-      return trigger;
-    }
-    if (
-      document.activeElement &&
-      typeof document.activeElement.focus === "function"
-    ) {
-      return document.activeElement;
-    }
-    return null;
-  }
-
   setNavigationTriggersExpanded(expanded) {
     if (typeof document.querySelectorAll !== "function") {
       return;
@@ -465,8 +182,8 @@ class NavigationSearch extends HTMLElement {
   }
 
   updateComboboxState() {
-    const dialog = this.shadowRoot.querySelector("dialog");
+    const dialog = this.querySelector("dialog");
-    const input = this.shadowRoot.querySelector(".search-input");
+    const input = this.querySelector(".search-input");
     const matches = this.renderedMatches || [];
     this.setElementAttribute(
       input,
@@ -491,7 +208,7 @@ class NavigationSearch extends HTMLElement {
   }
 
   setStatus(message) {
-    const status = this.shadowRoot.querySelector(`#${this.statusId}`);
+    const status = this.querySelector(`#${this.statusId}`);
     if (status) {
       status.textContent = message || "";
     }
@@ -701,7 +418,7 @@ class NavigationSearch extends HTMLElement {
       section.render(node, {
         navigationSearch: this,
         container,
-        input: this.shadowRoot.querySelector(".search-input"),
+        input: this.querySelector(".search-input"),
       });
     });
   }
@@ -740,8 +457,8 @@ class NavigationSearch extends HTMLElement {
   }
 
   renderResults() {
-    const container = this.shadowRoot.querySelector(".results-container");
+    const container = this.querySelector(".results-container");
-    const input = this.shadowRoot.querySelector(".search-input");
+    const input = this.querySelector(".search-input");
     const showStartContent = !input.value.trim();
     const jumpSections = showStartContent ? this.jumpSections() : [];
     const startBlock = showStartContent
@@ -853,18 +570,15 @@ class NavigationSearch extends HTMLElement {
     }
   }
 
-  openMenu(trigger) {
+  openMenu(returnFocusTo) {
-    const dialog = this.shadowRoot.querySelector("dialog");
+    const input = this.querySelector(".search-input");
-    const input = this.shadowRoot.querySelector(".search-input");
 
-    this.restoreFocusTarget = this.focusRestoreTarget(trigger);
+    this.querySelector("datasette-modal").show({
-    this.shouldRestoreFocus = true;
+      returnFocusTo,
-    if (!dialog.open) {
+      initialFocus: input,
-      dialog.showModal();
+    });
-    }
     this.setNavigationTriggersExpanded(true);
     input.value = "";
-    input.focus();
 
     // Reset state, then populate the default jump list.
     this.matches = [];
@@ -874,29 +588,15 @@ class NavigationSearch extends HTMLElement {
   }
 
   closeMenu(options = {}) {
-    const dialog = this.shadowRoot.querySelector("dialog");
+    this.querySelector("datasette-modal").close(options);
-    this.shouldRestoreFocus = options.restoreFocus !== false;
-    if (dialog.open) {
-      dialog.close();
-    } else {
-      this.onMenuClosed();
-    }
   }
 
   onMenuClosed() {
-    const input = this.shadowRoot.querySelector(".search-input");
+    const input = this.querySelector(".search-input");
     this.setElementAttribute(input, "aria-expanded", "false");
     this.removeElementAttribute(input, "aria-activedescendant");
     this.setNavigationTriggersExpanded(false);
     this.setStatus("");
-    if (
-      this.shouldRestoreFocus &&
-      this.restoreFocusTarget &&
-      typeof this.restoreFocusTarget.focus === "function"
-    ) {
-      this.restoreFocusTarget.focus();
-    }
-    this.restoreFocusTarget = null;
   }
 
   escapeHtml(text) {

datasette/static/table.js

@@ -157,6 +157,7 @@ function createSetColumnTypeOption(value, name, description, checked) {
 
 function setSetColumnTypeDialogBusy(state, isBusy) {
   state.isBusy = isBusy;
+  state.modal.busy = isBusy;
   state.saveButton.disabled = isBusy;
   state.cancelButton.disabled = isBusy;
   Array.from(
@@ -185,7 +186,8 @@ function ensureSetColumnTypeDialog() {
     return null;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.id = SET_COLUMN_TYPE_DIALOG_ID;
   dialog.className = "set-column-type-dialog";
   dialog.setAttribute("aria-labelledby", "set-column-type-title");
@@ -196,16 +198,17 @@ function ensureSetColumnTypeDialog() {
     </div>
     <p class="set-column-type-status"></p>
     <p class="set-column-type-error" hidden></p>
-    <div class="set-column-type-options"></div>
+    <div class="modal-body set-column-type-options"></div>
     <div class="modal-footer">
       <span class="footer-info"></span>
-      <button type="button" class="btn btn-ghost set-column-type-cancel">Cancel</button>
+      <button type="button" class="modal-btn modal-btn-ghost set-column-type-cancel">Cancel</button>
-      <button type="button" class="btn btn-primary set-column-type-save">Save</button>
+      <button type="button" class="modal-btn modal-btn-primary set-column-type-save">Save</button>
     </div>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   setColumnTypeDialogState = {
+    modal: modal,
     dialog: dialog,
     meta: dialog.querySelector(".modal-meta"),
     status: dialog.querySelector(".set-column-type-status"),
@@ -220,21 +223,7 @@ function ensureSetColumnTypeDialog() {
   };
 
   setColumnTypeDialogState.cancelButton.addEventListener("click", function () {
-    if (!setColumnTypeDialogState.isBusy) {
+    modal.requestClose("cancel");
-      dialog.close();
-    }
-  });
-
-  dialog.addEventListener("click", function (ev) {
-    if (ev.target === dialog && !setColumnTypeDialogState.isBusy) {
-      dialog.close();
-    }
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    if (setColumnTypeDialogState.isBusy) {
-      ev.preventDefault();
-    }
   });
 
   dialog.addEventListener("close", function () {
@@ -242,49 +231,52 @@ function ensureSetColumnTypeDialog() {
     setSetColumnTypeDialogBusy(setColumnTypeDialogState, false);
   });
 
-  setColumnTypeDialogState.saveButton.addEventListener("click", async function () {
+  setColumnTypeDialogState.saveButton.addEventListener(
-    var state = setColumnTypeDialogState;
+    "click",
-    var selected = state.dialog.querySelector(
+    async function () {
-      'input[name="set-column-type-choice"]:checked',
+      var state = setColumnTypeDialogState;
-    );
+      var selected = state.dialog.querySelector(
-    var selectedType = selected ? selected.value : "";
+        'input[name="set-column-type-choice"]:checked',
-    var currentType = state.currentConfig.current
+      );
-      ? state.currentConfig.current.type
+      var selectedType = selected ? selected.value : "";
-      : "";
+      var currentType = state.currentConfig.current
+        ? state.currentConfig.current.type
+        : "";
 
-    if (selectedType === currentType) {
+      if (selectedType === currentType) {
-      state.dialog.close();
+        state.modal.close();
-      return;
+        return;
-    }
-
-    clearSetColumnTypeDialogError(state);
-    setSetColumnTypeDialogBusy(state, true);
-
-    var payload = {
-      column: state.currentColumn,
-      column_type: selectedType ? { type: selectedType } : null,
-    };
-
-    try {
-      var response = await fetch(getSetColumnTypeData().path, {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-          Accept: "application/json",
-        },
-        body: JSON.stringify(payload),
-      });
-      var data = await response.json();
-      if (!response.ok || data.ok === false) {
-        var message = (data.errors || ["Request failed"]).join(" ");
-        throw new Error(message);
       }
-      location.reload();
+
-    } catch (error) {
+      clearSetColumnTypeDialogError(state);
-      setSetColumnTypeDialogBusy(state, false);
+      setSetColumnTypeDialogBusy(state, true);
-      showSetColumnTypeDialogError(state, error.message || "Request failed");
+
-    }
+      var payload = {
-  });
+        column: state.currentColumn,
+        column_type: selectedType ? { type: selectedType } : null,
+      };
+
+      try {
+        var response = await fetch(getSetColumnTypeData().path, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Accept: "application/json",
+          },
+          body: JSON.stringify(payload),
+        });
+        var data = await response.json();
+        if (!response.ok || data.ok === false) {
+          var message = (data.errors || ["Request failed"]).join(" ");
+          throw new Error(message);
+        }
+        location.reload();
+      } catch (error) {
+        setSetColumnTypeDialogBusy(state, false);
+        showSetColumnTypeDialogError(state, error.message || "Request failed");
+      }
+    },
+  );
 
   return setColumnTypeDialogState;
 }
@@ -341,9 +333,7 @@ function openSetColumnTypeDialog(th) {
     state.optionsWrap.appendChild(emptyState);
   }
 
-  if (!state.dialog.open) {
+  state.modal.show();
-    state.dialog.showModal();
-  }
   var selectedOption = state.dialog.querySelector(
     'input[name="set-column-type-choice"]:checked',
   );
@@ -367,9 +357,10 @@ function shouldShowShowAllColumns() {
 
 function hasMultipleVisibleColumns(manager) {
   return (
-    Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter(
+    Array.from(
-      (th) => th.dataset.column && th.dataset.isLinkColumn !== "1",
+      document.querySelectorAll(manager.selectors.tableHeaders),
-    ).length > 1
+    ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1")
+      .length > 1
   );
 }
 
@@ -649,10 +640,12 @@ function filterRowNumberFromName(name) {
 }
 
 function nextFilterRowNumber(manager) {
-  return filterRowsWithControls(manager).reduce((max, row) => {
+  return (
-    var column = row.querySelector("select");
+    filterRowsWithControls(manager).reduce((max, row) => {
-    return Math.max(max, filterRowNumberFromName(column && column.name));
+      var column = row.querySelector("select");
-  }, 0) + 1;
+      return Math.max(max, filterRowNumberFromName(column && column.name));
+    }, 0) + 1
+  );
 }
 
 function setFilterRowNumber(row, number) {
@@ -679,9 +672,11 @@ function updateFilterRowButtons(manager) {
     if (addButton) {
       addButton.hidden = index !== rows.length - 1 || !column.value;
     }
-    var visibleButtonCount = [removeButton, addButton].filter(function (button) {
+    var visibleButtonCount = [removeButton, addButton].filter(
-      return button && !button.hidden;
+      function (button) {
-    }).length;
+        return button && !button.hidden;
+      },
+    ).length;
     row.classList.toggle(
       "filter-controls-row-has-buttons",
       visibleButtonCount > 0,
@@ -703,7 +698,9 @@ function cloneFilterRow(row) {
   clone.querySelector(".filter-op select").name = "_filter_op";
   clone.querySelector("input.filter-value").name = "_filter_value";
   resetFilterRow(clone);
-  clone.querySelectorAll(".filter-row-icon").forEach((button) => button.remove());
+  clone
+    .querySelectorAll(".filter-row-icon")
+    .forEach((button) => button.remove());
   return clone;
 }
 
@@ -860,10 +857,45 @@ function openColumnChooser() {
   });
 }
 
+function initCountAll() {
+  var button = document.querySelector(".count-all");
+  if (!button) {
+    return;
+  }
+  button.addEventListener("click", async function () {
+    var count = document.querySelector(".table-count");
+    var error = document.querySelector(".count-error");
+    button.disabled = true;
+    button.textContent = "Counting…";
+    error.textContent = "";
+    try {
+      var response = await fetch(button.dataset.countUrl + location.search, {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+        },
+      });
+      var data = await response.json();
+      if (!response.ok || !data.ok) {
+        throw new Error((data.errors || ["Count failed"]).join(" "));
+      }
+      count.textContent =
+        data.count.toLocaleString("en-US") +
+        (data.count === 1 ? " row" : " rows");
+      button.remove();
+    } catch (ex) {
+      error.textContent = ex.message || "Count failed";
+      button.disabled = false;
+      button.textContent = "count all";
+    }
+  });
+}
+
 // Ensures Table UI is initialized only after the Manager is ready.
 document.addEventListener("datasette_init", function (evt) {
   const { detail: manager } = evt;
 
+  initCountAll();
   initializeColumnActions(manager);
 
   // Main table

datasette/stored_queries.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
-from dataclasses import dataclass
 import json
-from typing import Any, Iterable
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any
 
 from .utils import tilde_encode, urlsafe_components
 
@@ -386,7 +387,7 @@ async def count_queries(
                 OR q.sql LIKE :query_search
             )
             """)
-        params["query_search"] = "%{}%".format(q)
+        params["query_search"] = f"%{q}%"
     if is_write is not None:
         where_clauses.append("q.is_write = :query_is_write")
         params["query_is_write"] = int(bool(is_write))
@@ -462,7 +463,7 @@ async def list_queries(
         except ValueError:
             components = []
         if database is None and len(components) == 3:
-            where_clauses.append("""
+            where_clauses.append(f"""
                 (
                     q.database_name > :cursor_database
                     OR (
@@ -476,12 +477,12 @@ async def list_queries(
                         )
                     )
                 )
-                """.format(sort_key_sql=sort_key_sql))
+                """)
             params["cursor_database"] = components[0]
             params["cursor_sort_key"] = components[1]
             params["cursor_name"] = components[2]
         elif database is not None and len(components) == 2:
-            where_clauses.append("""
+            where_clauses.append(f"""
                 (
                     {sort_key_sql} > :cursor_sort_key
                     OR (
@@ -489,7 +490,7 @@ async def list_queries(
                         AND q.name > :cursor_name
                     )
                 )
-                """.format(sort_key_sql=sort_key_sql))
+                """)
             params["cursor_sort_key"] = components[0]
             params["cursor_name"] = components[1]
 
@@ -502,7 +503,7 @@ async def list_queries(
                 OR q.sql LIKE :query_search
             )
             """)
-        params["query_search"] = "%{}%".format(q)
+        params["query_search"] = f"%{q}%"
     if is_write is not None:
         where_clauses.append("q.is_write = :query_is_write")
         params["query_is_write"] = int(bool(is_write))

datasette/telemetry.py

@@ -0,0 +1,481 @@
+"""
+OpenTelemetry integration for Datasette.
+
+This uses `opentelemetry-api` only. Providers, exporters and sampling are
+configured by whoever runs Datasette, for example `opentelemetry-instrument`.
+"""
+
+import contextvars
+import re
+import threading
+import time
+import weakref
+from contextlib import contextmanager
+
+from opentelemetry import context as otel_context_api
+from opentelemetry import metrics as otel_metrics
+from opentelemetry import trace as otel_trace
+from opentelemetry.propagate import extract
+from opentelemetry.propagators.textmap import Getter
+from opentelemetry.trace import Link, SpanKind, Status, StatusCode, get_current_span
+
+from .telemetry_registry import (
+    DB_NAMESPACE,
+    DB_SYSTEM,
+    ERROR_TYPE,
+    HTTP_REQUEST_METHOD,
+    HTTP_RESPONSE_STATUS_CODE,
+    INTERNAL_CLIENT,
+    M_CONNECTIONS_OPEN,
+    M_OPERATION_DURATION,
+    M_QUERIES_INTERRUPTED,
+    M_QUERIES_PENDING,
+    M_THREADS_LIMIT,
+    M_THREADS_QUEUE_DEPTH,
+    M_WRITE_QUEUE_DEPTH,
+    M_WRITE_QUEUE_WAIT,
+    OPERATION,
+    SERVER_ADDRESS,
+    URL_PATH,
+    URL_SCHEME,
+    USER_AGENT_ORIGINAL,
+)
+from .version import __version__
+
+# True while code is executing within a datasette.client request. Defined
+# here rather than in app.py to avoid a circular import.
+_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False)
+
+# The semantic conventions version matching the attribute names used here.
+# 1.30.0 renamed `db.system` to `db.system.name`, so update this when
+# renaming attributes to match a newer version.
+SCHEMA_URL = "https://opentelemetry.io/schemas/1.29.0"
+
+tracer = otel_trace.get_tracer("datasette", __version__, schema_url=SCHEMA_URL)
+meter = otel_metrics.get_meter("datasette", __version__, schema_url=SCHEMA_URL)
+
+MAX_SQL_LENGTH = 2048
+
+
+def sql_attribute(sql: str) -> str:
+    "Truncate SQL text so it is safe to attach to a span as an attribute."
+    sql = sql.strip()
+    if len(sql) <= MAX_SQL_LENGTH:
+        return sql
+    return sql[:MAX_SQL_LENGTH] + "…[truncated]"
+
+
+def callback_name(fn) -> str:
+    """
+    The name recorded as `datasette.callback` for a callback-style call.
+
+    Falls back to the type name for callables such as `functools.partial`
+    that have no `__qualname__`.
+    """
+    return getattr(fn, "__qualname__", type(fn).__name__)
+
+
+def linked_root_span_kwargs(context=None):
+    """
+    Keyword arguments that start a new root span with a ``Link`` back to
+    the current span.
+
+    Use this for work that can outlive the span that caused it, such as a
+    background task or a ``block=False`` write.
+
+    Pass ``context`` to link to the span in a previously captured context
+    instead of the current one. If there is no valid span, no link is added.
+
+    Works with any tracer::
+
+        with my_tracer.start_as_current_span(
+            "myplugin.job", **linked_root_span_kwargs()
+        ):
+            ...
+    """
+    cause = get_current_span(context).get_span_context()
+    links = [Link(cause)] if cause.is_valid else []
+    return {"context": otel_context_api.Context(), "links": links}
+
+
+# Keywords that can be recorded as db.operation.name. SQL can be supplied by
+# users, so an allowlist keeps the number of distinct values small.
+DB_OPERATION_ALLOWLIST = frozenset(
+    {
+        "SELECT",
+        "INSERT",
+        "UPDATE",
+        "DELETE",
+        "CREATE",
+        "DROP",
+        "ALTER",
+        "PRAGMA",
+        "EXPLAIN",
+        "REPLACE",
+        "VACUUM",
+        "ANALYZE",
+        "WITH",
+    }
+)
+
+_LEADING_KEYWORD = re.compile(r"^\s*([A-Za-z]+)")
+
+
+def sql_operation_name(sql: str) -> str | None:
+    """
+    The statement's leading keyword if it is in the allowlist, else None.
+
+    Statements that start with a comment or "(" return None. Statements
+    starting with a CTE return `WITH`. Only call this for a single statement.
+    """
+    match = _LEADING_KEYWORD.match(sql)
+    if not match:
+        return None
+    keyword = match.group(1).upper()
+    if keyword in DB_OPERATION_ALLOWLIST:
+        return keyword
+    return None
+
+
+# --- The HTTP request span ------------------------------------------------
+
+
+class _ScopeHeadersGetter(Getter):
+    "Read W3C trace context from an ASGI scope's headers."
+
+    def get(self, carrier, key):
+        wanted = key.lower().encode("latin-1")
+        values = [v.decode("latin-1") for k, v in carrier if k.lower() == wanted]
+        return values or None
+
+    def keys(self, carrier):
+        return [k.decode("latin-1") for k, _ in carrier]
+
+
+_HEADERS_GETTER = _ScopeHeadersGetter()
+
+
+# Methods defined by RFC 9110 plus PATCH (RFC 5789). Anything else is
+# recorded as `_OTHER`, as recommended by semantic conventions.
+_KNOWN_METHODS = frozenset(
+    {"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"}
+)
+
+
+def clamp_http_method(method):
+    "The request method if it is one we recognise, else ``_OTHER``."
+    method = (method or "").upper()
+    return method if method in _KNOWN_METHODS else "_OTHER"
+
+
+def _first_header(headers, name):
+    "The first value of a header, decoded, or None."
+    for key, value in headers:
+        if key.lower() == name:
+            return value.decode("latin-1")
+    return None
+
+
+def _url_path(scope):
+    """
+    The request path, with any query string removed.
+
+    Prefers `raw_path`, which preserves encoded slashes in database and
+    table names. Some clients include the query string in `raw_path`, so
+    that is stripped as well.
+    """
+    raw_path = scope.get("raw_path")
+    if raw_path:
+        if isinstance(raw_path, bytes):
+            raw_path = raw_path.decode("latin-1")
+        return raw_path.split("?", 1)[0]
+    return scope.get("path", "")
+
+
+# The request span is passed to the router in the ASGI scope, because a
+# plugin's asgi_wrapper() middleware may have made its own span current.
+# Absent if the span is not recording.
+REQUEST_SPAN_SCOPE_KEY = "datasette.telemetry.request_span"
+
+
+def request_span(scope):
+    """
+    The recording request span for an ASGI scope, or None.
+
+    Falls back to the current span, for when Datasette is running under
+    other instrumentation.
+    """
+    span = scope.get(REQUEST_SPAN_SCOPE_KEY)
+    if span is None:
+        span = otel_trace.get_current_span()
+    return span if span.is_recording() else None
+
+
+class TelemetryMiddleware:
+    """
+    One `SpanKind.SERVER` span per HTTP request.
+
+    The span ends after the full response, including any streamed body,
+    has been sent.
+    """
+
+    def __init__(self, app):
+        self.app = app
+
+    async def __call__(self, scope, receive, send):
+        # Pass lifespan and websocket scopes straight through
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+        headers = scope.get("headers") or []
+        # Uses the global propagator, configured with OTEL_PROPAGATORS
+        context = extract(headers, getter=_HEADERS_GETTER)
+        method = clamp_http_method(scope.get("method", ""))
+        # Renamed to include the route once routing has happened
+        with tracer.start_as_current_span(
+            method, context=context, kind=SpanKind.SERVER
+        ) as span:
+            if not span.is_recording():
+                # No provider installed, or the trace was not sampled
+                await self.app(scope, receive, send)
+                return
+            span.set_attribute(HTTP_REQUEST_METHOD, method)
+            span.set_attribute(URL_PATH, _url_path(scope))
+            scheme = scope.get("scheme")
+            if scheme:
+                span.set_attribute(URL_SCHEME, scheme)
+            host = _first_header(headers, b"host")
+            if host:
+                span.set_attribute(SERVER_ADDRESS, host)
+            user_agent = _first_header(headers, b"user-agent")
+            if user_agent:
+                span.set_attribute(USER_AGENT_ORIGINAL, user_agent)
+            if _in_datasette_client.get():
+                span.set_attribute(INTERNAL_CLIENT, True)
+
+            scope = dict(scope, **{REQUEST_SPAN_SCOPE_KEY: span})
+
+            # Some responses are sent without a Response object, so the
+            # status is captured by wrapping send()
+            status_holder = {}
+
+            async def wrapped_send(message):
+                if (
+                    message["type"] == "http.response.start"
+                    and "status" not in status_holder
+                ):
+                    status_holder["status"] = message["status"]
+                await send(message)
+
+            escaped = False
+            try:
+                await self.app(scope, receive, wrapped_send)
+            except BaseException as exception:
+                # Includes asyncio.CancelledError when a client disconnects
+                escaped = True
+                span.set_attribute(ERROR_TYPE, type(exception).__name__)
+                span.set_status(Status(StatusCode.ERROR, str(exception)))
+                raise
+            finally:
+                status = status_holder.get("status")
+                if status is not None:
+                    span.set_attribute(HTTP_RESPONSE_STATUS_CODE, status)
+                    # 4xx responses are not errors for a server span. If an
+                    # exception escaped, keep its class name as error.type.
+                    if status >= 500 and not escaped:
+                        span.set_status(Status(StatusCode.ERROR))
+                        span.set_attribute(ERROR_TYPE, str(status))
+
+
+# --- Metrics --------------------------------------------------------------
+
+
+def _duration_attributes(database_name, operation):
+    return {
+        DB_SYSTEM: "sqlite",
+        DB_NAMESPACE: database_name,
+        OPERATION: operation,
+    }
+
+
+# Instruments use plain text descriptions. The registry entries have longer
+# reStructuredText descriptions for the documentation.
+
+sql_operation_duration = meter.create_histogram(
+    M_OPERATION_DURATION,
+    unit=M_OPERATION_DURATION.unit,
+    description="Duration of a SQL operation issued by Datasette",
+    explicit_bucket_boundaries_advisory=M_OPERATION_DURATION.buckets,
+)
+
+write_queue_wait = meter.create_histogram(
+    M_WRITE_QUEUE_WAIT,
+    unit=M_WRITE_QUEUE_WAIT.unit,
+    description=(
+        "Time a write spent queued behind the single write thread for its database"
+    ),
+    explicit_bucket_boundaries_advisory=M_WRITE_QUEUE_WAIT.buckets,
+)
+
+queries_interrupted = meter.create_counter(
+    M_QUERIES_INTERRUPTED,
+    unit=M_QUERIES_INTERRUPTED.unit,
+    description="Queries cancelled for exceeding sql_time_limit_ms",
+)
+
+
+@contextmanager
+def record_operation_duration(database_name, operation):
+    """
+    Record `db.client.operation.duration` for one SQL operation.
+
+    Sets `error.type` to the exception class on failure. For a `block=False`
+    write this measures the time taken to enqueue the write.
+    """
+    attributes = _duration_attributes(database_name, operation)
+    started = time.perf_counter()
+    try:
+        yield
+    except BaseException as exception:
+        attributes[ERROR_TYPE] = type(exception).__qualname__
+        raise
+    finally:
+        sql_operation_duration.record(time.perf_counter() - started, attributes)
+
+
+def record_write_queue_wait(database_name, waited_ns):
+    write_queue_wait.record(waited_ns / 1e9, {DB_NAMESPACE: database_name})
+
+
+def record_query_interrupted(database_name):
+    queries_interrupted.add(1, {DB_NAMESPACE: database_name})
+
+
+# Live Datasette instances reported by the gauges below. The lock is needed
+# because gauge callbacks run on the SDK's collection thread.
+#
+# The pool gauges do not identify which instance they came from, so they
+# are only meaningful for a process running a single Datasette instance.
+_live_datasettes = weakref.WeakSet()
+_live_datasettes_lock = threading.Lock()
+
+
+def register_datasette(ds):
+    "Start reporting pool/queue gauges for this Datasette instance."
+    with _live_datasettes_lock:
+        _live_datasettes.add(ds)
+
+
+def unregister_datasette(ds):
+    "Stop reporting gauges for an instance that has been closed."
+    with _live_datasettes_lock:
+        _live_datasettes.discard(ds)
+
+
+def _live_instances():
+    with _live_datasettes_lock:
+        return list(_live_datasettes)
+
+
+def _databases_of(ds):
+    "Every Database attached to an instance, including the internal database."
+    databases = list(ds.databases.values())
+    internal = getattr(ds, "_internal_database", None)
+    if internal is not None:
+        databases.append(internal)
+    return databases
+
+
+def observe_sql_thread_limit(options=None):
+    "Size of the shared read-query thread pool (the num_sql_threads setting)."
+    for ds in _live_instances():
+        if ds.executor is None:
+            # num_sql_threads=0 - queries run on the event loop, no pool.
+            continue
+        yield otel_metrics.Observation(ds.setting("num_sql_threads"), {})
+
+
+def observe_sql_thread_queue_depth(options=None):
+    """
+    Read queries waiting for a free thread in the shared pool.
+
+    `_work_queue` is a private attribute of ThreadPoolExecutor, so this
+    reports nothing if it is missing.
+    """
+    for ds in _live_instances():
+        if ds.executor is None:
+            continue
+        work_queue = getattr(ds.executor, "_work_queue", None)
+        if work_queue is None:
+            continue
+        yield otel_metrics.Observation(work_queue.qsize(), {})
+
+
+def observe_pending_queries(options=None):
+    """
+    Read queries submitted to the pool and not yet finished, per database.
+
+    Reads `len()` without `_pending_execute_futures_lock` to avoid blocking
+    queries.
+    """
+    for ds in _live_instances():
+        for db in _databases_of(ds):
+            yield otel_metrics.Observation(
+                len(db._pending_execute_futures), {DB_NAMESPACE: db.name}
+            )
+
+
+def observe_write_queue_depth(options=None):
+    "Writes queued behind the single write thread, per database."
+    for ds in _live_instances():
+        for db in _databases_of(ds):
+            write_queue = db._write_queue
+            if write_queue is None:
+                # No write has ever been queued for this database.
+                continue
+            yield otel_metrics.Observation(write_queue.qsize(), {DB_NAMESPACE: db.name})
+
+
+def observe_open_connections(options=None):
+    "Open SQLite connections tracked for closing, per database."
+    for ds in _live_instances():
+        for db in _databases_of(ds):
+            yield otel_metrics.Observation(
+                len(db._all_connections), {DB_NAMESPACE: db.name}
+            )
+
+
+sql_thread_limit_gauge = meter.create_observable_gauge(
+    M_THREADS_LIMIT,
+    callbacks=[observe_sql_thread_limit],
+    unit=M_THREADS_LIMIT.unit,
+    description="Maximum concurrent read queries (the num_sql_threads setting)",
+)
+
+sql_thread_queue_depth_gauge = meter.create_observable_gauge(
+    M_THREADS_QUEUE_DEPTH,
+    callbacks=[observe_sql_thread_queue_depth],
+    unit=M_THREADS_QUEUE_DEPTH.unit,
+    description="Read queries waiting for a free thread in the shared SQL pool",
+)
+
+pending_queries_gauge = meter.create_observable_gauge(
+    M_QUERIES_PENDING,
+    callbacks=[observe_pending_queries],
+    unit=M_QUERIES_PENDING.unit,
+    description="Read queries submitted to the pool and not yet complete",
+)
+
+write_queue_depth_gauge = meter.create_observable_gauge(
+    M_WRITE_QUEUE_DEPTH,
+    callbacks=[observe_write_queue_depth],
+    unit=M_WRITE_QUEUE_DEPTH.unit,
+    description="Writes queued behind a database's single write thread",
+)
+
+open_connections_gauge = meter.create_observable_gauge(
+    M_CONNECTIONS_OPEN,
+    callbacks=[observe_open_connections],
+    unit=M_CONNECTIONS_OPEN.unit,
+    description="Open SQLite connections tracked for closing",
+)

datasette/telemetry_registry.py

@@ -0,0 +1,502 @@
+"""
+Every span, metric and attribute that Datasette emits.
+
+These entries are used by the instrumentation code, by `docs/telemetry_doc.py`
+to generate the documentation, and by `tests/test_telemetry_registry.py` to
+check that the emitted telemetry matches the registry.
+"""
+
+from opentelemetry.trace import SpanKind
+
+
+class Attribute(str):
+    """
+    A span attribute key, carrying its own documentation.
+
+    Subclasses `str` so it can be handed straight to `set_attribute()`.
+
+    Part of Datasette's public plugin API - plugins declare their own
+    telemetry registries with these classes. See the "Telemetry for plugin
+    authors" documentation.
+    """
+
+    __slots__ = ("description", "optional", "values")
+
+    def __new__(cls, name, description, optional=False, values=None):
+        self = super().__new__(cls, name)
+        self.description = description
+        self.optional = optional
+        # The allowed values for this attribute, or None to allow any value
+        self.values = frozenset(values) if values is not None else None
+        return self
+
+    def __reduce__(self):
+        # Copies and pickles become a plain str, since __new__ requires the
+        # extra arguments. ConsoleMetricExporter deepcopies attribute keys.
+        return (str, (str(self),))
+
+    def __repr__(self):
+        return f"Attribute({str(self)!r})"
+
+
+class SpanName(str):
+    """A span name, carrying its documentation and the attributes it may set.
+
+    Part of Datasette's public plugin API, like `Attribute`.
+    """
+
+    __slots__ = ("attributes", "description", "dynamic", "kind", "prefix")
+
+    def __new__(
+        cls,
+        name,
+        description,
+        attributes=(),
+        prefix=False,
+        dynamic=False,
+        kind=SpanKind.INTERNAL,
+    ):
+        self = super().__new__(cls, name)
+        self.description = description
+        self.attributes = tuple(attributes)
+        # Match emitted names that start with this prefix, for names with a
+        # variable suffix such as SpanName("chat ", ..., prefix=True)
+        self.prefix = prefix
+        # The emitted name is built at runtime, so `span_for()` matches it by
+        # span kind. The entry's string is a template for the documentation.
+        self.dynamic = dynamic
+        self.kind = kind
+        return self
+
+    def __reduce__(self):
+        # See Attribute.__reduce__.
+        return (str, (str(self),))
+
+    def __repr__(self):
+        return f"SpanName({str(self)!r})"
+
+
+class MetricName(str):
+    "A metric name, carrying its instrument kind, unit and attributes."
+
+    __slots__ = ("attributes", "buckets", "description", "kind", "unit")
+
+    def __new__(cls, name, kind, unit, description, attributes=(), buckets=None):
+        self = super().__new__(cls, name)
+        self.kind = kind
+        self.unit = unit
+        self.description = description
+        self.attributes = tuple(attributes)
+        # Explicit bucket boundaries, for histograms only
+        self.buckets = tuple(buckets) if buckets is not None else None
+        return self
+
+    def __reduce__(self):
+        # See Attribute.__reduce__.
+        return (str, (str(self),))
+
+    def __repr__(self):
+        return f"MetricName({str(self)!r})"
+
+
+COUNTER = "Counter"
+UPDOWN_COUNTER = "UpDownCounter"
+HISTOGRAM = "Histogram"
+GAUGE = "Observable gauge"
+
+
+# --- Attributes -----------------------------------------------------------
+
+HTTP_REQUEST_METHOD = Attribute(
+    "http.request.method",
+    "The HTTP request method. Methods outside the nine defined by RFC 9110 "
+    "and RFC 5789 are recorded as ``_OTHER``.",
+)
+HTTP_RESPONSE_STATUS_CODE = Attribute(
+    "http.response.status_code",
+    "The HTTP response status code. Omitted if no response was started.",
+    optional=True,
+)
+HTTP_ROUTE = Attribute(
+    "http.route",
+    "The regular expression for the matched route, for example "
+    "``/(?P<database>[^\\/\\.]+)/(?P<table>[^\\/\\.]+)(\\.(?P<format>\\w+))?$`` "
+    "for a table page. Use this attribute to group requests by route. "
+    "Omitted when no route matches.",
+    optional=True,
+)
+URL_PATH = Attribute(
+    "url.path",
+    "The URL path, excluding the query string.",
+)
+URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.")
+SERVER_ADDRESS = Attribute(
+    "server.address",
+    "The ``Host`` header, including any ``:port`` suffix. This value is "
+    "supplied by the client.",
+    optional=True,
+)
+USER_AGENT_ORIGINAL = Attribute(
+    "user_agent.original",
+    "The ``User-Agent`` header, verbatim. Omitted if the client sent none.",
+    optional=True,
+)
+INTERNAL_CLIENT = Attribute(
+    "datasette.internal_client",
+    "``True`` for requests made through ``datasette.client``. Calls made "
+    "inside another request produce a nested ``SERVER`` span. Filter on "
+    "this attribute to exclude internal requests from request counts. "
+    "Omitted for requests received over the network.",
+    optional=True,
+)
+ERROR_TYPE = Attribute(
+    "error.type",
+    "The exception class name for a failed operation. On HTTP spans, also "
+    "set to the status code as a string for 5xx responses. A 4xx response "
+    "alone does not set this attribute or an error status.",
+    optional=True,
+)
+
+DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.")
+DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.")
+OPERATION = Attribute(
+    "datasette.operation",
+    "Whether the operation was a read or a write.",
+    values={"read", "write"},
+)
+DB_QUERY_TEXT = Attribute(
+    "db.query.text",
+    "The SQL, truncated to 2048 characters. Bound parameter values are not "
+    "recorded. For callback methods, ``datasette.callback`` is recorded instead.",
+    optional=True,
+)
+CALLBACK = Attribute(
+    "datasette.callback",
+    "The qualified name of the Python callable passed to ``execute_fn()``, "
+    "``execute_write_fn()`` or ``execute_isolated_fn()``, for example "
+    "``TableInsertView.post.<locals>.insert_or_upsert_rows``. Set instead of "
+    "``db.query.text``. Lambdas appear as ``<lambda>``; use a named function "
+    "for a more descriptive span.",
+    optional=True,
+)
+DB_OPERATION_NAME = Attribute(
+    "db.operation.name",
+    "The statement's leading keyword, such as ``SELECT``, ``INSERT`` or "
+    "``CREATE``, if it matches the supported allowlist. Statements beginning "
+    "with a common table expression report ``WITH``. Omitted for unrecognized "
+    "keywords and ``execute_write_script()``.",
+    optional=True,
+)
+PARAM_COUNT = Attribute(
+    "datasette.param_count",
+    "Number of bound parameters. Recorded instead of the values themselves.",
+    optional=True,
+)
+PARAM_SETS = Attribute(
+    "datasette.param_sets",
+    "Number of parameter sets consumed by ``execute_write_many()``. "
+    "The parameter values are not recorded.",
+    optional=True,
+)
+TIME_LIMIT_MS = Attribute(
+    "datasette.time_limit_ms",
+    "Time limit applied to the read query, in milliseconds: "
+    ":ref:`setting_sql_time_limit_ms` or a shorter ``custom_time_limit``.",
+    optional=True,
+)
+ROWS_RETURNED = Attribute(
+    "datasette.rows_returned",
+    "Number of rows returned by a successful read query.",
+    optional=True,
+)
+TRUNCATED = Attribute(
+    "datasette.truncated",
+    "True if the result was cut short by :ref:`setting_max_returned_rows`.",
+    optional=True,
+)
+INTERRUPTED = Attribute(
+    "datasette.interrupted",
+    "True if the query exceeded its time limit. The span status is set to "
+    "``ERROR`` unless the caller used a ``custom_time_limit`` shorter than "
+    ":ref:`setting_sql_time_limit_ms`, in which case the status is left unset.",
+    optional=True,
+)
+SQL_ERROR_SUPPRESSED = Attribute(
+    "datasette.sql_error_suppressed",
+    "True for a non-timeout SQL error with ``log_sql_errors=False``. The "
+    "exception is still raised, but the span status is left unset.",
+    optional=True,
+)
+EXECUTESCRIPT = Attribute(
+    "datasette.executescript",
+    "True for ``execute_write_script()``, which runs multiple statements.",
+    optional=True,
+)
+EXECUTEMANY = Attribute(
+    "datasette.executemany",
+    "True for ``execute_write_many()``, which runs one statement against many "
+    "parameter sets.",
+    optional=True,
+)
+ISOLATED_CONNECTION = Attribute(
+    "datasette.isolated_connection",
+    "True if the write ran on its own connection rather than the shared write "
+    "connection.",
+)
+TRANSACTION = Attribute(
+    "datasette.transaction",
+    "False for statements such as ``VACUUM`` that cannot run inside a transaction.",
+)
+
+
+# --- Spans ----------------------------------------------------------------
+
+HTTP_REQUEST = SpanName(
+    "{http.request.method} {http.route}",
+    "One span per HTTP request, containing spans from plugin middleware and "
+    "database operations. Named for the HTTP method and matched route, or "
+    "just the method if no route matches. Incoming ``traceparent`` headers "
+    "are extracted using the global propagator to continue the caller's "
+    "trace. Incoming ``baggage`` is not propagated into plugin or downstream "
+    "context in this release. Set ``OTEL_PROPAGATORS=none`` to disable "
+    "extraction. For public instances, strip trace context headers at your "
+    "proxy if callers should not supply trace context.",
+    (
+        HTTP_REQUEST_METHOD,
+        HTTP_ROUTE,
+        URL_PATH,
+        URL_SCHEME,
+        SERVER_ADDRESS,
+        USER_AGENT_ORIGINAL,
+        HTTP_RESPONSE_STATUS_CODE,
+        ERROR_TYPE,
+        INTERNAL_CLIENT,
+    ),
+    dynamic=True,
+    kind=SpanKind.SERVER,
+)
+
+DB_QUERY = SpanName(
+    "db.query",
+    "A SQL operation, including time spent queued for a worker thread. For "
+    "``block=False`` writes, the span ends after the write is queued. "
+    "Callback methods record ``datasette.callback`` in place of ``db.query.text``.",
+    (
+        DB_SYSTEM,
+        DB_NAMESPACE,
+        DB_QUERY_TEXT,
+        CALLBACK,
+        DB_OPERATION_NAME,
+        PARAM_COUNT,
+        PARAM_SETS,
+        TIME_LIMIT_MS,
+        ROWS_RETURNED,
+        TRUNCATED,
+        INTERRUPTED,
+        SQL_ERROR_SUPPRESSED,
+        EXECUTESCRIPT,
+        EXECUTEMANY,
+    ),
+    kind=SpanKind.CLIENT,
+)
+
+DB_QUERY_EXECUTE = SpanName(
+    "db.query.execute",
+    "The read executing inside a SQL worker thread. Child of ``db.query``; the "
+    "gap between the two is time spent waiting for a thread.",
+)
+
+DB_WRITE_QUEUE_WAIT = SpanName(
+    "db.write.queue_wait",
+    "Time a write spent waiting in its database's write queue. For "
+    "``block=True``, this is a child of ``db.query``. For ``block=False``, "
+    "it is a root span linked to the span that queued the write, since the "
+    "write can outlive that request.",
+)
+
+DB_WRITE_EXECUTE = SpanName(
+    "db.write.execute",
+    "The write executing on the write thread. For ``block=True``, this is "
+    "a child of ``db.query``. For ``block=False``, it is a root span linked "
+    "to the span that queued the write.",
+    (ISOLATED_CONNECTION, TRANSACTION),
+)
+
+STARTUP = SpanName(
+    "datasette.startup",
+    "Startup work performed by ``invoke_startup()``, including registration "
+    "hooks, schema catalog updates, saved queries, column type configuration "
+    "and the ``startup`` hook. Runs during instance startup, either before "
+    "serving requests or as part of the first request.",
+)
+
+SPANS = (
+    HTTP_REQUEST,
+    DB_QUERY,
+    DB_QUERY_EXECUTE,
+    DB_WRITE_QUEUE_WAIT,
+    DB_WRITE_EXECUTE,
+    STARTUP,
+)
+
+
+def span_for(emitted_name, kind=None, spans=None):
+    """
+    Resolve an emitted span name to its registry entry, or None.
+
+    Exact matches take precedence over `prefix=True` entries, which take
+    precedence over `dynamic=True` entries matched by `kind`.
+
+    `spans` defaults to Datasette's own registry.
+    """
+    if spans is None:
+        spans = SPANS
+    for span in spans:
+        if span.dynamic:
+            continue
+        if emitted_name == span:
+            return span
+    for span in spans:
+        if span.prefix and emitted_name.startswith(span):
+            return span
+    if kind is not None:
+        for span in spans:
+            if span.dynamic and span.kind == kind:
+                return span
+    return None
+
+
+def metric_for(emitted_name, metrics=None):
+    """
+    Resolve an emitted metric name to its registry entry, or None.
+
+    `metrics` defaults to Datasette's own registry.
+    """
+    if metrics is None:
+        metrics = METRICS
+    for metric in metrics:
+        if emitted_name == metric:
+            return metric
+    return None
+
+
+def attribute_allowed(entry, emitted_key):
+    """
+    Whether `emitted_key` is a registered attribute of `entry`.
+
+    `entry` is a `SpanName` or a `MetricName` - both carry `.attributes`.
+    """
+    if entry is None:
+        return False
+    return emitted_key in entry.attributes
+
+
+def attribute_value_allowed(entry, emitted_key, value):
+    """
+    Whether `value` is permitted for `emitted_key` on `entry` (a `SpanName`
+    or a `MetricName`).
+
+    Any value is allowed if the attribute does not declare `values=`.
+    """
+    if entry is None:
+        return False
+    for attribute in entry.attributes:
+        if attribute == emitted_key:
+            return attribute.values is None or value in attribute.values
+    return False
+
+
+# --- Metrics --------------------------------------------------------------
+
+# Bucket boundaries in seconds for every duration histogram. OpenTelemetry's
+# defaults are designed for milliseconds and would put almost every SQLite
+# query in the first bucket. These are the semantic conventions' recommended
+# boundaries for db.client.operation.duration, plus 0.0001 and 0.0005 for
+# fast in-process SQLite queries.
+DURATION_BUCKETS = (0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10)
+
+M_OPERATION_DURATION = MetricName(
+    "db.client.operation.duration",
+    HISTOGRAM,
+    "s",
+    "Duration of a SQL operation, including callback-based calls such as "
+    "``execute_fn()``. For ``block=False`` writes, measures enqueue time.",
+    (DB_SYSTEM, DB_NAMESPACE, OPERATION, ERROR_TYPE),
+    buckets=DURATION_BUCKETS,
+)
+
+M_WRITE_QUEUE_WAIT = MetricName(
+    "datasette.write.queue_wait",
+    HISTOGRAM,
+    "s",
+    "Time each write waited in its database's write queue.",
+    (DB_NAMESPACE,),
+    buckets=DURATION_BUCKETS,
+)
+
+M_QUERIES_INTERRUPTED = MetricName(
+    "datasette.sql.queries.interrupted",
+    COUNTER,
+    "{query}",
+    "Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. A "
+    "rising rate can indicate that queries need optimization or a higher "
+    "time limit. Caller-selected timeouts shorter than this limit, such as "
+    "those used for facet suggestion, are excluded.",
+    (DB_NAMESPACE,),
+)
+
+M_THREADS_LIMIT = MetricName(
+    "datasette.sql.threads.limit",
+    GAUGE,
+    "{thread}",
+    "Maximum concurrent read queries, configured by "
+    ":ref:`setting_num_sql_threads`. Not reported when ``num_sql_threads`` "
+    "is ``0``.",
+)
+
+M_THREADS_QUEUE_DEPTH = MetricName(
+    "datasette.sql.threads.queue_depth",
+    GAUGE,
+    "{query}",
+    "Read queries waiting for a free SQL thread. Sustained values above "
+    "zero indicate a saturated read pool.",
+)
+
+M_QUERIES_PENDING = MetricName(
+    "datasette.sql.queries.pending",
+    GAUGE,
+    "{query}",
+    "Read queries submitted to the pool and not yet complete. Sum across "
+    "databases and compare with ``datasette.sql.threads.limit`` to assess "
+    "pool usage.",
+    (DB_NAMESPACE,),
+)
+
+M_WRITE_QUEUE_DEPTH = MetricName(
+    "datasette.write.queue_depth",
+    GAUGE,
+    "{write}",
+    "Writes waiting for a database's single write thread. Increasing "
+    "``num_sql_threads`` does not increase write concurrency. Not reported for "
+    "databases that have never been written to.",
+    (DB_NAMESPACE,),
+)
+
+M_CONNECTIONS_OPEN = MetricName(
+    "datasette.connections.open",
+    GAUGE,
+    "{connection}",
+    "Open SQLite connections managed by Datasette.",
+    (DB_NAMESPACE,),
+)
+
+METRICS = (
+    M_OPERATION_DURATION,
+    M_WRITE_QUEUE_WAIT,
+    M_QUERIES_INTERRUPTED,
+    M_THREADS_LIMIT,
+    M_THREADS_QUEUE_DEPTH,
+    M_QUERIES_PENDING,
+    M_WRITE_QUEUE_DEPTH,
+    M_CONNECTIONS_OPEN,
+)

datasette/telemetry_testing.py

@@ -0,0 +1,427 @@
+"""
+Pytest helpers for testing OpenTelemetry instrumentation - Datasette's own
+and any plugin's. Part of Datasette's public plugin API; see the "Telemetry
+for plugin authors" documentation.
+
+Usage from a plugin's ``conftest.py``::
+
+    from datasette.telemetry_testing import (  # noqa: F401
+        MetricsCollector,
+        otel_metrics,
+        otel_meter_provider,
+        otel_provider,
+        otel_spans,
+    )
+
+Tests can then use the ``otel_spans`` and ``otel_metrics`` fixtures. The
+OpenTelemetry SDK is imported lazily, and the fixtures skip if it is not
+installed.
+"""
+
+import subprocess
+import sys
+
+import pytest
+
+from .telemetry_registry import (
+    attribute_allowed,
+    attribute_value_allowed,
+    metric_for,
+    span_for,
+)
+
+_span_exporter = None
+_metric_reader = None
+
+
+def install_span_exporter():
+    """
+    Install a TracerProvider + InMemorySpanExporter once per process and
+    return the exporter, or None when the SDK is not installed.
+
+    Uses `SimpleSpanProcessor` so spans are exported as soon as they end.
+    """
+    global _span_exporter
+    if _span_exporter is not None:
+        return _span_exporter
+    try:
+        from opentelemetry import trace as otel_trace
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+        from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+            InMemorySpanExporter,
+        )
+    except ImportError:
+        return None
+    exporter = InMemorySpanExporter()
+    provider = TracerProvider()
+    provider.add_span_processor(SimpleSpanProcessor(exporter))
+    otel_trace.set_tracer_provider(provider)
+    # set_tracer_provider() is ignored if a provider was already installed,
+    # in which case the fixtures skip
+    if otel_trace.get_tracer_provider() is not provider:
+        return None
+    _span_exporter = exporter
+    return exporter
+
+
+def install_metric_reader():
+    """
+    Install a MeterProvider + InMemoryMetricReader once per process and
+    return the reader, or None when the SDK is not installed.
+
+    Uses delta temporality for counters and histograms, so each collection
+    only reports measurements since the previous one.
+    """
+    global _metric_reader
+    if _metric_reader is not None:
+        return _metric_reader
+    try:
+        from opentelemetry import metrics as otel_metrics_api
+        from opentelemetry.sdk.metrics import Counter, Histogram, MeterProvider
+        from opentelemetry.sdk.metrics.export import (
+            AggregationTemporality,
+            InMemoryMetricReader,
+        )
+    except ImportError:
+        return None
+    reader = InMemoryMetricReader(
+        preferred_temporality={
+            Counter: AggregationTemporality.DELTA,
+            Histogram: AggregationTemporality.DELTA,
+        }
+    )
+    provider = MeterProvider(metric_readers=[reader])
+    otel_metrics_api.set_meter_provider(provider)
+    if otel_metrics_api.get_meter_provider() is not provider:
+        return None
+    _metric_reader = reader
+    return reader
+
+
+@pytest.fixture(scope="session", autouse=True)
+def otel_provider():
+    "Install the span exporter once per test session, before any spans are created."
+    install_span_exporter()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def otel_meter_provider():
+    "Install the metric reader once per test session."
+    install_metric_reader()
+
+
+@pytest.fixture(autouse=True)
+def otel_reset():
+    "Clear recorded spans and drain collected metrics after every test."
+    yield
+    if _span_exporter is not None:
+        _span_exporter.clear()
+    if _metric_reader is not None:
+        _metric_reader.get_metrics_data()
+
+
+@pytest.fixture
+def otel_spans():
+    """
+    The in-memory span exporter, cleared before the test. Call
+    `.get_finished_spans()` to retrieve spans.
+    """
+    pytest.importorskip("opentelemetry.sdk")
+    exporter = install_span_exporter()
+    if exporter is None:
+        pytest.skip("OpenTelemetry SDK provider was not installed")
+    exporter.clear()
+    yield exporter
+
+
+class MetricsCollector:
+    """
+    Wraps an `InMemoryMetricReader`.
+
+    `collect()` runs a collection cycle and stores a snapshot, which
+    `points()` and `point()` then query.
+    """
+
+    def __init__(self, reader):
+        self.reader = reader
+        self.snapshot = {}
+        # (instrumentation scope name, sdk Metric) pairs from the last collect()
+        self.collected = []
+
+    def collect(self):
+        self.snapshot = {}
+        self.collected = []
+        data = self.reader.get_metrics_data()
+        if data is None:
+            return self.snapshot
+        for resource_metrics in data.resource_metrics:
+            for scope_metrics in resource_metrics.scope_metrics:
+                scope_name = scope_metrics.scope.name if scope_metrics.scope else None
+                for metric in scope_metrics.metrics:
+                    self.snapshot.setdefault(metric.name, []).extend(
+                        metric.data.data_points
+                    )
+                    self.collected.append((scope_name, metric))
+        return self.snapshot
+
+    def points(self, name, attributes=None):
+        "Data points for `name` whose attributes are a superset of `attributes`."
+        found = []
+        for point in self.snapshot.get(name, []):
+            point_attributes = dict(point.attributes or {})
+            if all(point_attributes.get(k) == v for k, v in (attributes or {}).items()):
+                found.append(point)
+        return found
+
+    def point(self, name, attributes=None):
+        "The single matching data point, asserting there is exactly one."
+        found = self.points(name, attributes)
+        assert len(found) == 1, (
+            f"expected exactly one {name} point matching {attributes}, "
+            f"got {len(found)}: {found}"
+        )
+        return found[0]
+
+
+@pytest.fixture
+def otel_metrics():
+    "A `MetricsCollector`, drained before the test so counts start from zero."
+    pytest.importorskip("opentelemetry.sdk")
+    reader = install_metric_reader()
+    if reader is None:
+        pytest.skip("OpenTelemetry SDK meter provider was not installed")
+    reader.get_metrics_data()
+    yield MetricsCollector(reader)
+
+
+def _scoped(finished_spans, scope_name):
+    if scope_name is None:
+        return list(finished_spans)
+    return [
+        span
+        for span in finished_spans
+        if span.instrumentation_scope and span.instrumentation_scope.name == scope_name
+    ]
+
+
+def assert_spans_conform(registry_spans, finished_spans, scope_name=None):
+    """
+    Assert every finished span is registered in `registry_spans`, sets only
+    registered attributes and uses allowed attribute values.
+
+    Pass `scope_name` to only check spans from that instrumentation scope.
+    """
+    problems = []
+    for span in _scoped(finished_spans, scope_name):
+        entry = span_for(str(span.name), kind=span.kind, spans=registry_spans)
+        if entry is None:
+            problems.append(f"unregistered span: {span.name!r}")
+            continue
+        for key, value in (span.attributes or {}).items():
+            if not attribute_allowed(entry, str(key)):
+                problems.append(f"{span.name}: unregistered attribute {key!r}")
+            elif not attribute_value_allowed(entry, str(key), value):
+                problems.append(
+                    f"{span.name}: {key}={value!r} not in the declared enum"
+                )
+    assert not problems, "\n".join(problems)
+
+
+def assert_spans_covered(registry_spans, finished_spans, scope_name=None):
+    """
+    Assert every entry in `registry_spans` was emitted at least once, with
+    each of its attributes that is not `optional=True`.
+    """
+    spans = _scoped(finished_spans, scope_name)
+    seen_attributes = {}
+    for span in spans:
+        entry = span_for(str(span.name), kind=span.kind, spans=registry_spans)
+        if entry is not None:
+            seen = seen_attributes.setdefault(str(entry), set())
+            seen.update(str(key) for key in (span.attributes or {}))
+    problems = []
+    for entry in registry_spans:
+        if str(entry) not in seen_attributes:
+            problems.append(f"registered span never emitted: {entry!r}")
+            continue
+        required = {
+            str(attribute) for attribute in entry.attributes if not attribute.optional
+        }
+        missing = required - seen_attributes[str(entry)]
+        if missing:
+            problems.append(
+                f"{entry}: registered attributes never emitted: {sorted(missing)}"
+            )
+    assert not problems, "\n".join(problems)
+
+
+# Registry instrument kinds mapped to the SDK data type collected for them.
+# Both counter kinds collect as Sum, distinguished by is_monotonic.
+_KIND_TO_DATA_TYPE = {
+    "Counter": "Sum",
+    "UpDownCounter": "Sum",
+    "Histogram": "Histogram",
+    "Observable gauge": "Gauge",
+}
+_KIND_IS_MONOTONIC = {"Counter": True, "UpDownCounter": False}
+
+
+def _scoped_metrics(collector, scope_name):
+    for scope, metric in collector.collected:
+        if scope_name is None or scope == scope_name:
+            yield metric
+
+
+def assert_metrics_conform(registry_metrics, collector, scope_name=None):
+    """
+    Assert every metric in the collector's last `collect()` is registered in
+    `registry_metrics` with a matching instrument kind and unit, sets only
+    registered attributes and uses allowed attribute values.
+
+    Pass `scope_name` to only check metrics from that instrumentation scope.
+    """
+    problems = set()
+    for metric in _scoped_metrics(collector, scope_name):
+        entry = metric_for(metric.name, metrics=registry_metrics)
+        if entry is None:
+            problems.add(f"unregistered metric: {metric.name!r}")
+            continue
+        expected_data_type = _KIND_TO_DATA_TYPE.get(entry.kind)
+        actual_data_type = type(metric.data).__name__
+        if expected_data_type is not None and actual_data_type != expected_data_type:
+            problems.add(
+                f"{metric.name}: registry declares {entry.kind}, "
+                f"SDK collected {actual_data_type}"
+            )
+        expected_monotonic = _KIND_IS_MONOTONIC.get(entry.kind)
+        actual_monotonic = getattr(metric.data, "is_monotonic", None)
+        if (
+            expected_monotonic is not None
+            and actual_monotonic is not None
+            and actual_monotonic != expected_monotonic
+        ):
+            problems.add(
+                f"{metric.name}: registry declares {entry.kind}, but the "
+                f"collected Sum is_monotonic={actual_monotonic}"
+            )
+        if (metric.unit or "") != (entry.unit or ""):
+            problems.add(
+                f"{metric.name}: instrument unit {metric.unit!r} != "
+                f"registry unit {entry.unit!r}"
+            )
+        for point in metric.data.data_points:
+            for key, value in dict(point.attributes or {}).items():
+                if not attribute_allowed(entry, str(key)):
+                    problems.add(f"{metric.name}: unregistered attribute {key!r}")
+                elif not attribute_value_allowed(entry, str(key), value):
+                    problems.add(
+                        f"{metric.name}: {key}={value!r} not in the declared enum"
+                    )
+    assert not problems, "\n".join(sorted(problems))
+
+
+def assert_metrics_covered(registry_metrics, collector, scope_name=None):
+    """
+    Assert every entry in `registry_metrics` was collected at least once,
+    with each of its attributes that is not `optional=True`.
+
+    Call `collect()` once after the workload and before this check.
+    """
+    seen_attributes = {}
+    for metric in _scoped_metrics(collector, scope_name):
+        entry = metric_for(metric.name, metrics=registry_metrics)
+        if entry is None:
+            continue
+        seen = seen_attributes.setdefault(str(entry), set())
+        for point in metric.data.data_points:
+            seen.update(str(key) for key in dict(point.attributes or {}))
+    problems = []
+    for entry in registry_metrics:
+        if str(entry) not in seen_attributes:
+            problems.append(f"registered metric never collected: {entry!r}")
+            continue
+        required = {
+            str(attribute) for attribute in entry.attributes if not attribute.optional
+        }
+        missing = required - seen_attributes[str(entry)]
+        if missing:
+            problems.append(
+                f"{entry}: registered attributes never collected: {sorted(missing)}"
+            )
+    assert not problems, "\n".join(problems)
+
+
+def assert_no_forbidden_values(
+    forbidden, finished_spans=None, collector=None, scope_name=None
+):
+    """
+    Assert that none of the `forbidden` strings appear anywhere in the
+    emitted telemetry: span names, span attribute values, span event names
+    and attributes, span status descriptions, or metric point attributes.
+
+    Use fake private values such as tokens or email addresses in your test
+    workload, then check that they were not recorded:
+
+        FORBIDDEN = {"secret-token-123", "alice@example.com"}
+        run_workload_using_those_values()
+        assert_no_forbidden_values(
+            FORBIDDEN,
+            finished_spans=otel_spans.get_finished_spans(),
+            collector=otel_metrics,
+        )
+
+    Matches substrings of each value's string form. Empty strings in
+    `forbidden` are ignored. Leave `scope_name` unset to also check
+    Datasette's own telemetry.
+    """
+    needles = [needle for needle in forbidden if needle]
+    leaks = set()
+
+    def check(value, where):
+        text = str(value)
+        for needle in needles:
+            if needle in text:
+                leaks.add(f"{where} contains {needle!r}")
+
+    if finished_spans is not None:
+        for span in _scoped(finished_spans, scope_name):
+            check(span.name, f"span name {str(span.name)!r}")
+            for key, value in (span.attributes or {}).items():
+                check(value, f"{span.name} attribute {key}")
+            for event in span.events or ():
+                check(event.name, f"{span.name} event name")
+                for key, value in (event.attributes or {}).items():
+                    check(value, f"{span.name} event {event.name} attribute {key}")
+            if span.status is not None and span.status.description:
+                check(span.status.description, f"{span.name} status description")
+    if collector is not None:
+        for metric in _scoped_metrics(collector, scope_name):
+            for point in metric.data.data_points:
+                for key, value in dict(point.attributes or {}).items():
+                    check(value, f"metric {metric.name} attribute {key}")
+    assert not leaks, "forbidden values leaked into telemetry:\n" + "\n".join(
+        sorted(leaks)
+    )
+
+
+def assert_package_never_imports_sdk(*module_names):
+    """
+    Import the named modules in a fresh interpreter and assert none of them
+    imported `opentelemetry.sdk`.
+
+    Run the test that calls this early in your suite: on macOS with CPython
+    3.13, starting a subprocess from a process with many threads can crash.
+    """
+    imports = "; ".join(f"import {name}" for name in module_names)
+    code = (
+        f"import sys; {imports}; "
+        "print([m for m in sys.modules if m.startswith('opentelemetry.sdk')])"
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", code], capture_output=True, text=True, check=True
+    )
+    assert result.stdout.strip() == "[]", (
+        f"importing {module_names} pulled in the OpenTelemetry SDK: "
+        f"{result.stdout.strip()}"
+    )

datasette/templates/_permission_ui_styles.html

@@ -6,8 +6,20 @@
     padding: 1.5em;
     margin-bottom: 2em;
 }
+.permission-form form {
+    max-width: 60rem;
+}
+.permission-form-grid {
+    display: grid;
+    gap: 1.5rem;
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+}
+.permission-form-result {
+    margin-top: 1rem;
+    max-width: 60rem;
+}
 .form-section {
-    margin-bottom: 1em;
+    margin-bottom: 1.25em;
 }
 .form-section label {
     display: block;
@@ -15,22 +27,51 @@
     font-weight: bold;
 }
 .form-section input[type="text"],
-.form-section select {
+.form-section input[type="number"],
-    width: 100%;
+.form-section select,
-    max-width: 500px;
+.permission-textarea {
-    padding: 0.5em;
+    background-color: #fff;
+    border: 1px solid #aaa;
+    border-radius: 4px;
     box-sizing: border-box;
-    border: 1px solid #ccc;
+    box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.08);
-    border-radius: 3px;
+    color: #222;
+    font-family: inherit;
+    font-size: 1rem;
+    line-height: 1.4;
+    max-width: none;
+    width: 100%;
+}
+.form-section input[type="text"] {
+    height: 3rem;
+    padding: 0.6rem 0.75rem;
+}
+.form-section input[type="number"] {
+    height: 3rem;
+    max-width: 7rem;
+    padding: 0.6rem 0.75rem;
+}
+.form-section select {
+    height: 3rem;
+    padding: 0.6rem 0.75rem;
+}
+.permission-textarea {
+    font-family: monospace;
+    min-height: 12rem;
+    padding: 0.75rem;
+    resize: vertical;
 }
 .form-section input[type="text"]:focus,
-.form-section select:focus {
+.form-section input[type="number"]:focus,
-    outline: 2px solid #0066cc;
+.form-section select:focus,
+.permission-textarea:focus {
     border-color: #0066cc;
+    box-shadow: 0 0 0 3px rgba(0, 102, 204, 0.18);
+    outline: none;
 }
 .form-section small {
     display: block;
-    margin-top: 0.3em;
+    margin-top: 0.45em;
     color: #666;
 }
 .form-actions {
@@ -142,4 +183,9 @@
     text-align: center;
     color: #666;
 }
+@media only screen and (max-width: 576px) {
+    .permission-form-grid {
+        grid-template-columns: minmax(0, 1fr);
+    }
+}
 </style>

datasette/templates/_permissions_debug_tabs.html

@@ -44,10 +44,10 @@
 </style>
 
 <nav class="permissions-debug-tabs">
-    <a href="{{ urls.path('-/permissions') }}" {% if current_tab == "permissions" %}class="active"{% endif %}>Playground</a>
+    <a href="{{ urls.path('-/check') }}{{ query_string }}" {% if current_tab == "check" %}class="active"{% endif %}>Explain</a>
-    <a href="{{ urls.path('-/check') }}{{ query_string }}" {% if current_tab == "check" %}class="active"{% endif %}>Check</a>
+    <a href="{{ urls.path('-/allowed') }}{{ query_string }}" {% if current_tab == "allowed" %}class="active"{% endif %}>Access map</a>
-    <a href="{{ urls.path('-/allowed') }}{{ query_string }}" {% if current_tab == "allowed" %}class="active"{% endif %}>Allowed</a>
+    <a href="{{ urls.path('-/rules') }}{{ query_string }}" {% if current_tab == "rules" %}class="active"{% endif %}>Rule explorer</a>
-    <a href="{{ urls.path('-/rules') }}{{ query_string }}" {% if current_tab == "rules" %}class="active"{% endif %}>Rules</a>
+    <a href="{{ urls.path('-/permissions') }}" {% if current_tab == "permissions" %}class="active"{% endif %}>Activity</a>
     <a href="{{ urls.path('-/actions') }}" {% if current_tab == "actions" %}class="active"{% endif %}>Actions</a>
     <a href="{{ urls.path('-/allow-debug') }}" {% if current_tab == "allow_debug" %}class="active"{% endif %}>Allow debug</a>
 </nav>

datasette/templates/allow_debug.html

@@ -3,29 +3,11 @@
 {% block title %}Debug allow rules{% endblock %}
 
 {% block extra_head %}
+{% include "_permission_ui_styles.html" %}
 <style>
-textarea {
-    height: 10em;
-    width: 95%;
-    box-sizing: border-box;
-    padding: 0.5em;
-    border: 2px dotted black;
-}
-.two-col {
-    display: inline-block;
-    width: 48%;
-}
-.two-col label {
-    width: 48%;
-}
 p.message-warning {
     white-space: pre-wrap;
 }
-@media only screen and (max-width: 576px) {
-    .two-col {
-        width: 100%;
-    }
-}
 </style>
 {% endblock %}
 
@@ -38,24 +20,28 @@ p.message-warning {
 
 <p>Use this tool to try out different actor and allow combinations. See <a href="https://docs.datasette.io/en/stable/authentication.html#defining-permissions-with-allow-blocks">Defining permissions with "allow" blocks</a> for documentation.</p>
 
-<form class="core" action="{{ urls.path('-/allow-debug') }}" method="get" style="margin-bottom: 1em">
+<div class="permission-form">
-    <div class="two-col">
+    <form class="core" action="{{ urls.path('-/allow-debug') }}" method="get">
-        <p><label>Allow block</label></p>
+        <div class="permission-form-grid">
-        <textarea name="allow">{{ allow_input }}</textarea>
+            <div class="form-section">
-    </div>
+                <label for="allow-block">Allow block</label>
-    <div class="two-col">
+                <textarea class="permission-textarea" id="allow-block" name="allow">{{ allow_input }}</textarea>
-        <p><label>Actor</label></p>
+            </div>
-        <textarea name="actor">{{ actor_input }}</textarea>
+            <div class="form-section">
-    </div>
+                <label for="allow-actor">Actor</label>
-    <div style="margin-top: 1em;">
+                <textarea class="permission-textarea" id="allow-actor" name="actor">{{ actor_input }}</textarea>
-        <input type="submit" value="Apply allow block to actor">
+            </div>
-    </div>
+        </div>
-</form>
+        <div class="form-actions">
+            <button type="submit" class="submit-btn">Apply allow block to actor</button>
+        </div>
+    </form>
 
-{% if error %}<p class="message-warning">{{ error }}</p>{% endif %}
+    {% if error %}<p class="message-warning permission-form-result">{{ error }}</p>{% endif %}
 
-{% if result == "True" %}<p class="message-info">Result: allow</p>{% endif %}
+    {% if result == "True" %}<p class="message-info permission-form-result">Result: allow</p>{% endif %}
 
-{% if result == "False" %}<p class="message-error">Result: deny</p>{% endif %}
+    {% if result == "False" %}<p class="message-error permission-form-result">Result: deny</p>{% endif %}
+</div>
 
 {% endblock %}

datasette/templates/api_explorer.html

@@ -3,7 +3,6 @@
 {% block title %}API Explorer{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% endblock %}
 
 {% block content %}
@@ -126,7 +125,7 @@ getForm.addEventListener("submit", (ev) => {
     document.getElementById('response-status').textContent = response.status;
     return response.json();
   }).then((data) => {
-    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
+    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
     errorList.style.display = 'none';
   }).catch((error) => {
     alert(error);
@@ -174,7 +173,7 @@ postForm.addEventListener("submit", (ev) => {
     } else {
       errorList.style.display = 'none';
     }
-    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
+    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
     output.style.display = 'block';
   }).catch(err => {
     alert("Error: " + err);

datasette/templates/base.html

@@ -8,6 +8,7 @@
     <link rel="stylesheet" href="{{ url.url }}"{% if url.get("sri") %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}>
 {% endfor %}
 <script>window.datasetteVersion = '{{ datasette_version }}';</script>
+<script src="{{ static('modal.js') }}" defer></script>
 <script src="{{ static('datasette-manager.js') }}" defer></script>
 {% for url in extra_js_urls %}
     <script {% if url.module %}type="module" {% endif %}src="{{ url.url }}"{% if url.get("sri") %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}></script>

datasette/templates/debug_allowed.html

@@ -3,7 +3,6 @@
 {% block title %}Allowed Resources{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -49,7 +48,7 @@
 
         <div class="form-section">
             <label for="page_size">Page size:</label>
-            <input type="number" id="page_size" name="_size" value="50" min="1" max="200" style="max-width: 100px;">
+            <input type="number" id="page_size" name="_size" value="50" min="1" max="200">
             <small>Number of results per page (max 200)</small>
         </div>
 
@@ -198,7 +197,7 @@ function displayResults(data) {
     }
 
     // Update raw JSON
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayError(data) {
@@ -208,7 +207,7 @@ function displayError(data) {
 
     resultsContent.innerHTML = `<div class="error-message">Error: ${escapeHtml(data.error || 'Unknown error')}</div>`;
 
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 // Disable child input if parent is empty

datasette/templates/debug_check.html

@@ -1,9 +1,8 @@
 {% extends "base.html" %}
 
-{% block title %}Permission Check{% endblock %}
+{% block title %}Explain a permission decision{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 <style>
@@ -13,29 +12,35 @@
     border-radius: 5px;
 }
 #output.allowed {
-    background-color: #e8f5e9;
+    background-color: #f3fbf4;
     border: 2px solid #4caf50;
 }
 #output.denied {
-    background-color: #ffebee;
+    background-color: #fff7f7;
     border: 2px solid #f44336;
 }
 #output h2 {
     margin-top: 0;
 }
-#output .result-badge {
+#output h3 {
+    margin-bottom: 0.5em;
+}
+#output .result-badge,
+.effect-badge,
+.rule-status {
     display: inline-block;
-    padding: 0.3em 0.8em;
+    padding: 0.2em 0.5em;
     border-radius: 3px;
     font-weight: bold;
-    font-size: 1.1em;
 }
-#output .allowed-badge {
+#output .allowed-badge,
-    background-color: #4caf50;
+.effect-allow {
+    background-color: #2e7d32;
     color: white;
 }
-#output .denied-badge {
+#output .denied-badge,
-    background-color: #f44336;
+.effect-deny {
+    background-color: #c62828;
     color: white;
 }
 .details-section {
@@ -48,70 +53,130 @@
 .details-section dd {
     margin-left: 1em;
 }
+.explanation-section {
+    background: rgba(255, 255, 255, 0.75);
+    border: 1px solid #ddd;
+    border-radius: 4px;
+    margin-top: 1em;
+    padding: 0 1em 1em;
+}
+.rules-table {
+    border-collapse: collapse;
+    width: 100%;
+}
+.rules-table th,
+.rules-table td {
+    border-bottom: 1px solid #ddd;
+    padding: 0.5em;
+    text-align: left;
+    vertical-align: top;
+}
+.rule-status {
+    background: #e8f5e9;
+    color: #1b5e20;
+}
+.rule-ignored {
+    background: #eee;
+    color: #555;
+    font-weight: normal;
+}
+.requirement-allowed {
+    color: #1b5e20;
+}
+.requirement-denied {
+    color: #b71c1c;
+}
+@media only screen and (max-width: 576px) {
+    .rules-table,
+    .rules-table tbody,
+    .rules-table tr,
+    .rules-table td {
+        display: block;
+    }
+    .rules-table thead {
+        display: none;
+    }
+    .rules-table td::before {
+        content: attr(data-label) ": ";
+        font-weight: bold;
+    }
+}
 </style>
 {% endblock %}
 
 {% block content %}
-<h1>Permission check</h1>
+<h1>Explain a permission decision</h1>
 
 {% set current_tab = "check" %}
 {% include "_permissions_debug_tabs.html" %}
 
-<p>Use this tool to test permission checks for the current actor. It queries the <code>/-/check.json</code> API endpoint.</p>
+<p>Test an actor, action and resource. The result explains which rules matched, which specificity level won, and whether actor restrictions or required actions changed the verdict.</p>
-
-{% if request.actor %}
-<p>Current actor: <strong>{{ request.actor.get("id", "anonymous") }}</strong></p>
-{% else %}
-<p>Current actor: <strong>anonymous (not logged in)</strong></p>
-{% endif %}
 
 <div class="permission-form">
-    <form id="check-form" method="get" action="{{ urls.path("-/check") }}">
+    <form id="check-form" method="get" action="{{ urls.path('-/check') }}">
         <div class="form-section">
-            <label for="action">Action (permission name):</label>
+            <label for="actor">Actor JSON:</label>
+            <textarea class="permission-textarea" id="actor" name="actor">{{ actor_json }}</textarea>
+            <small>Use <code>null</code> for an anonymous actor. This actor is simulated; it does not change who you are signed in as.</small>
+        </div>
+
+        <div class="form-section">
+            <label for="action">Action:</label>
             <select id="action" name="action" required>
                 <option value="">Select an action...</option>
-                {% for action_name in sorted_actions %}
+                {% for action in actions %}
-                <option value="{{ action_name }}">{{ action_name }}</option>
+                <option value="{{ action.name }}">{{ action.name }}{% if action.description %} — {{ action.description }}{% endif %}</option>
                 {% endfor %}
             </select>
-            <small>The permission action to check</small>
+            <small id="action-help">The operation to evaluate</small>
         </div>
 
-        <div class="form-section">
+        <div class="form-section" id="parent-section">
-            <label for="parent">Parent resource (optional):</label>
+            <label for="parent">Parent resource:</label>
             <input type="text" id="parent" name="parent" placeholder="e.g., database name">
-            <small>For database-level permissions, specify the database name</small>
+            <small>The database or other parent resource</small>
         </div>
 
-        <div class="form-section">
+        <div class="form-section" id="child-section">
-            <label for="child">Child resource (optional):</label>
+            <label for="child">Child resource:</label>
-            <input type="text" id="child" name="child" placeholder="e.g., table name">
+            <input type="text" id="child" name="child" placeholder="e.g., table or query name">
-            <small>For table-level permissions, specify the table name (requires parent)</small>
+            <small>The table, query or other child resource</small>
         </div>
 
         <div class="form-actions">
-            <button type="submit" class="submit-btn" id="submit-btn">Check Permission</button>
+            <button type="submit" class="submit-btn" id="submit-btn">Explain decision</button>
         </div>
     </form>
 </div>
 
 <div id="output" style="display: none;">
     <h2>Result: <span class="result-badge" id="result-badge"></span></h2>
+    <p id="result-summary"></p>
 
     <dl class="details-section">
+        <dt>Actor:</dt>
+        <dd><code id="result-actor"></code></dd>
         <dt>Action:</dt>
-        <dd id="result-action"></dd>
+        <dd><code id="result-action"></code></dd>
-
+        <dt>Resource:</dt>
-        <dt>Resource Path:</dt>
+        <dd><code id="result-resource"></code></dd>
-        <dd id="result-resource"></dd>
-
-        <dt>Actor ID:</dt>
-        <dd id="result-actor"></dd>
-
-        <div id="additional-details"></div>
     </dl>
 
+    <section class="explanation-section">
+        <h3>Matching rules</h3>
+        <div id="matching-rules"></div>
+    </section>
+
+    <section class="explanation-section" id="restrictions-section">
+        <h3>Actor restrictions</h3>
+        <div id="restriction-results"></div>
+    </section>
+
+    <section class="explanation-section" id="requirements-section">
+        <h3>Required actions</h3>
+        <div id="requirement-results"></div>
+    </section>
+
     <details style="margin-top: 1em;">
         <summary style="cursor: pointer; font-weight: bold;">Raw JSON response</summary>
         <pre id="raw-json" style="margin-top: 1em; padding: 1em; background-color: #f5f5f5; border: 1px solid #ddd; border-radius: 3px; overflow-x: auto;"></pre>
@@ -119,152 +184,134 @@
 </div>
 
 <script>
+const actions = Object.fromEntries({{ actions|tojson }}.map(action => [action.name, action]));
 const form = document.getElementById('check-form');
 const output = document.getElementById('output');
 const submitBtn = document.getElementById('submit-btn');
+const actionSelect = document.getElementById('action');
+
+function updateResourceFields() {
+    const action = actions[actionSelect.value];
+    document.getElementById('parent-section').style.display = action && action.takes_parent ? 'block' : 'none';
+    document.getElementById('child-section').style.display = action && action.takes_child ? 'block' : 'none';
+    let help = action && action.description ? action.description : 'The operation to evaluate';
+    if (action && action.also_requires) {
+        help += `; also requires ${action.also_requires}`;
+    }
+    document.getElementById('action-help').textContent = help;
+}
 
 async function performCheck() {
     submitBtn.disabled = true;
-    submitBtn.textContent = 'Checking...';
+    submitBtn.textContent = 'Explaining...';
-
+    const params = new URLSearchParams(new FormData(form));
-    const formData = new FormData(form);
-    const params = new URLSearchParams();
-
-    for (const [key, value] of formData.entries()) {
-        if (value) {
-            params.append(key, value);
-        }
-    }
 
     try {
         const response = await fetch('{{ urls.path("-/check.json") }}?' + params.toString(), {
-            method: 'GET',
+            headers: {'Accept': 'application/json'}
-            headers: {
-                'Accept': 'application/json',
-            }
         });
-
         const data = await response.json();
-
         if (response.ok) {
             displayResult(data);
         } else {
             displayError(data);
         }
     } catch (error) {
-        alert('Error: ' + error.message);
+        displayError({error: error.message});
     } finally {
         submitBtn.disabled = false;
-        submitBtn.textContent = 'Check Permission';
+        submitBtn.textContent = 'Explain decision';
     }
 }
 
-// Populate form on initial load
-(function() {
-    const params = populateFormFromURL();
-    const action = params.get('action');
-    if (action) {
-        performCheck();
-    }
-})();
-
 function displayResult(data) {
     output.style.display = 'block';
-
-    // Set badge and styling
     const resultBadge = document.getElementById('result-badge');
-    if (data.allowed) {
+    output.className = data.allowed ? 'allowed' : 'denied';
-        output.className = 'allowed';
+    resultBadge.className = `result-badge ${data.allowed ? 'allowed-badge' : 'denied-badge'}`;
-        resultBadge.className = 'result-badge allowed-badge';
+    resultBadge.textContent = data.allowed ? 'ALLOWED ✓' : 'DENIED ✗';
-        resultBadge.textContent = 'ALLOWED ✓';
+    document.getElementById('result-summary').textContent = data.explanation.summary;
-    } else {
+    document.getElementById('result-actor').textContent = data.actor === null ? 'anonymous' : JSON.stringify(data.actor);
-        output.className = 'denied';
+    document.getElementById('result-action').textContent = data.action;
-        resultBadge.className = 'result-badge denied-badge';
+    document.getElementById('result-resource').textContent = data.resource.path;
-        resultBadge.textContent = 'DENIED ✗';
+    displayRules(data.explanation);
+    displayRestrictions(data.explanation.restrictions);
+    displayRequirements(data.explanation.required_actions);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
+}
+
+function displayRules(explanation) {
+    const container = document.getElementById('matching-rules');
+    if (!explanation.matched_rules.length) {
+        container.innerHTML = '<p>No rules matched. Datasette denies access when there is no matching rule.</p>';
+        return;
     }
-
+    let html = '<table class="rules-table"><thead><tr><th>Effect</th><th>Scope</th><th>Source</th><th>Reason</th><th>Role in decision</th></tr></thead><tbody>';
-    // Basic details
+    for (const rule of explanation.matched_rules) {
-    document.getElementById('result-action').textContent = data.action || 'N/A';
+        const status = rule.decisive
-    document.getElementById('result-resource').textContent = data.resource?.path || '/';
+            ? '<span class="rule-status">Decisive</span>'
-    document.getElementById('result-actor').textContent = data.actor_id || 'anonymous';
+            : `<span class="rule-status rule-ignored">${escapeHtml(rule.ignored_because)}</span>`;
-
+        html += '<tr>';
-    // Additional details
+        html += `<td data-label="Effect"><span class="effect-badge effect-${rule.effect}">${rule.effect.toUpperCase()}</span></td>`;
-    const additionalDetails = document.getElementById('additional-details');
+        html += `<td data-label="Scope">${escapeHtml(rule.scope)}</td>`;
-    additionalDetails.innerHTML = '';
+        html += `<td data-label="Source"><code>${escapeHtml(rule.source || 'unknown')}</code></td>`;
-
+        html += `<td data-label="Reason">${escapeHtml(rule.reason || 'No reason supplied')}</td>`;
-    if (data.reason !== undefined) {
+        html += `<td data-label="Role in decision">${status}</td>`;
-        const dt = document.createElement('dt');
+        html += '</tr>';
-        dt.textContent = 'Reason:';
-        const dd = document.createElement('dd');
-        dd.textContent = data.reason || 'N/A';
-        additionalDetails.appendChild(dt);
-        additionalDetails.appendChild(dd);
     }
+    container.innerHTML = html + '</tbody></table>';
+}
 
-    if (data.source_plugin !== undefined) {
+function displayRestrictions(restrictions) {
-        const dt = document.createElement('dt');
+    const section = document.getElementById('restrictions-section');
-        dt.textContent = 'Source Plugin:';
+    const container = document.getElementById('restriction-results');
-        const dd = document.createElement('dd');
+    section.style.display = restrictions.length ? 'block' : 'none';
-        dd.textContent = data.source_plugin || 'N/A';
+    container.innerHTML = restrictions.map(restriction => {
-        additionalDetails.appendChild(dt);
+        const className = restriction.allowed ? 'requirement-allowed' : 'requirement-denied';
-        additionalDetails.appendChild(dd);
+        const verdict = restriction.allowed ? 'INCLUDED ✓' : 'EXCLUDED ✗';
-    }
+        return `<p class="${className}"><strong>${verdict}</strong> by <code>${escapeHtml(restriction.source || 'unknown')}</code>: ${escapeHtml(restriction.reason)}</p>`;
+    }).join('');
+}
 
-    if (data.used_default !== undefined) {
+function displayRequirements(requirements) {
-        const dt = document.createElement('dt');
+    const section = document.getElementById('requirements-section');
-        dt.textContent = 'Used Default:';
+    const container = document.getElementById('requirement-results');
-        const dd = document.createElement('dd');
+    section.style.display = requirements.length ? 'block' : 'none';
-        dd.textContent = data.used_default ? 'Yes' : 'No';
+    container.innerHTML = requirements.map(requirement => {
-        additionalDetails.appendChild(dt);
+        const className = requirement.allowed ? 'requirement-allowed' : 'requirement-denied';
-        additionalDetails.appendChild(dd);
+        const verdict = requirement.allowed ? 'ALLOWED ✓' : 'DENIED ✗';
-    }
+        return `<p class="${className}"><strong>${escapeHtml(requirement.action)}: ${verdict}</strong> — ${escapeHtml(requirement.summary)}</p>`;
-
+    }).join('');
-    if (data.depth !== undefined) {
-        const dt = document.createElement('dt');
-        dt.textContent = 'Depth:';
-        const dd = document.createElement('dd');
-        dd.textContent = data.depth;
-        additionalDetails.appendChild(dt);
-        additionalDetails.appendChild(dd);
-    }
-
-    // Raw JSON
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
-
-    // Scroll to output
-    output.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
 }
 
 function displayError(data) {
     output.style.display = 'block';
     output.className = 'denied';
-
     const resultBadge = document.getElementById('result-badge');
     resultBadge.className = 'result-badge denied-badge';
     resultBadge.textContent = 'ERROR';
-
+    document.getElementById('result-summary').textContent = data.error || 'Unknown error';
-    document.getElementById('result-action').textContent = 'N/A';
+    document.getElementById('result-actor').textContent = '—';
-    document.getElementById('result-resource').textContent = 'N/A';
+    document.getElementById('result-action').textContent = '—';
-    document.getElementById('result-actor').textContent = 'N/A';
+    document.getElementById('result-resource').textContent = '—';
-
+    document.getElementById('matching-rules').innerHTML = '';
-    const additionalDetails = document.getElementById('additional-details');
+    document.getElementById('restrictions-section').style.display = 'none';
-    additionalDetails.innerHTML = '<dt>Error:</dt><dd>' + (data.error || 'Unknown error') + '</dd>';
+    document.getElementById('requirements-section').style.display = 'none';
-
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
-
-    output.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
 }
 
-// Disable child input if parent is empty
+form.addEventListener('submit', event => {
-const parentInput = document.getElementById('parent');
+    event.preventDefault();
-const childInput = document.getElementById('child');
+    performCheck();
-
-childInput.addEventListener('focus', () => {
-    if (!parentInput.value) {
-        alert('Please specify a parent resource first before adding a child resource.');
-        parentInput.focus();
-    }
 });
-</script>
+actionSelect.addEventListener('change', updateResourceFields);
 
+(function initializeFromUrl() {
+    const params = populateFormFromURL();
+    updateResourceFields();
+    if (params.get('action')) {
+        performCheck();
+    }
+})();
+</script>
 {% endblock %}

datasette/templates/debug_permissions_playground.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 
-{% block title %}Debug permissions{% endblock %}
+{% block title %}Permission activity{% endblock %}
 
 {% block extra_head %}
 {% include "_permission_ui_styles.html" %}
@@ -20,60 +20,45 @@
 .check-action, .check-when, .check-result {
     font-size: 1.3em;
 }
-textarea {
-    height: 10em;
-    width: 95%;
-    box-sizing: border-box;
-    padding: 0.5em;
-    border: 2px dotted black;
-}
-.two-col {
-    display: inline-block;
-    width: 48%;
-}
-.two-col label {
-    width: 48%;
-}
-@media only screen and (max-width: 576px) {
-    .two-col {
-        width: 100%;
-    }
-}
 </style>
 {% endblock %}
 
 {% block content %}
-<h1>Permission playground</h1>
+<h1>Permission activity</h1>
 
 {% set current_tab = "permissions" %}
 {% include "_permissions_debug_tabs.html" %}
 
-<p>This tool lets you simulate an actor and a permission check for that actor.</p>
+<h2>Raw simulator</h2>
+
+<p>This form runs a hypothetical permission check and returns its raw explanation JSON. Use the <a href="{{ urls.path('-/check') }}">Explain tool</a> for a visual explanation of the same decision.</p>
 
 <div class="permission-form">
     <form action="{{ urls.path('-/permissions') }}" id="debug-post" method="post">
-        <div class="two-col">
+        <div class="permission-form-grid">
-            <div class="form-section">
+            <div>
-                <label>Actor</label>
+                <div class="form-section">
-                <textarea name="actor">{% if actor_input %}{{ actor_input }}{% else %}{"id": "root"}{% endif %}</textarea>
+                    <label for="activity-actor">Actor</label>
+                    <textarea class="permission-textarea" id="activity-actor" name="actor">{% if actor_input %}{{ actor_input }}{% else %}{"id": "root"}{% endif %}</textarea>
+                </div>
             </div>
-        </div>
+            <div>
-        <div class="two-col" style="vertical-align: top">
+                <div class="form-section">
-            <div class="form-section">
+                    <label for="permission">Action</label>
-                <label for="permission">Action</label>
+                    <select name="permission" id="permission">
-                <select name="permission" id="permission">
+                        {% for permission in permissions %}
-                    {% for permission in permissions %}
+                            <option value="{{ permission.name }}">{{ permission.name }}</option>
-                        <option value="{{ permission.name }}">{{ permission.name }}</option>
+                        {% endfor %}
-                    {% endfor %}
+                    </select>
-                </select>
+                </div>
-            </div>
+                <div class="form-section">
-            <div class="form-section">
+                    <label for="resource_1">Parent</label>
-                <label for="resource_1">Parent</label>
+                    <input type="text" id="resource_1" name="resource_1" placeholder="e.g., database name">
-                <input type="text" id="resource_1" name="resource_1" placeholder="e.g., database name">
+                </div>
-            </div>
+                <div class="form-section">
-            <div class="form-section">
+                    <label for="resource_2">Child</label>
-                <label for="resource_2">Child</label>
+                    <input type="text" id="resource_2" name="resource_2" placeholder="e.g., table name">
-                <input type="text" id="resource_2" name="resource_2" placeholder="e.g., table name">
+                </div>
             </div>
         </div>
         <div class="form-actions">
@@ -125,7 +110,7 @@ debugPost.addEventListener('submit', function(ev) {
 });
 </script>
 
-<h1>Recent permissions checks</h1>
+<h2>Recent permission checks</h2>
 
 <p>
     {% if filter != "all" %}<a href="?filter=all">All</a>{% else %}<strong>All</strong>{% endif %},

datasette/templates/debug_rules.html

@@ -3,7 +3,6 @@
 {% block title %}Permission Rules{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -37,7 +36,7 @@
 
         <div class="form-section">
             <label for="page_size">Page size:</label>
-            <input type="number" id="page_size" name="_size" value="50" min="1" max="200" style="max-width: 100px;">
+            <input type="number" id="page_size" name="_size" value="50" min="1" max="200">
             <small>Number of results per page (max 200)</small>
         </div>
 
@@ -185,7 +184,7 @@ function displayResults(data) {
     }
 
     // Update raw JSON
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayError(data) {
@@ -195,7 +194,7 @@ function displayError(data) {
 
     resultsContent.innerHTML = `<div class="error-message">Error: ${escapeHtml(data.error || 'Unknown error')}</div>`;
 
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 </script>

datasette/templates/index.html

@@ -26,8 +26,7 @@
             {% if database.show_table_row_counts %}{{ "{:,}".format(database.hidden_table_rows_sum) }} rows in {% endif %}{{ database.hidden_tables_count }} hidden table{% if database.hidden_tables_count != 1 %}s{% endif -%}
         {% endif -%}
         {% if database.views_count -%}
-            {% if database.tables_count or database.hidden_tables_count %}, {% endif -%}
+            , {{ "{:,}".format(database.views_count) }} view{% if database.views_count != 1 %}s{% endif %}
-            {{ "{:,}".format(database.views_count) }} view{% if database.views_count != 1 %}s{% endif %}
         {% endif %}
     </p>
     <p>{% for table in database.tables_and_views_truncated %}<a href="{{ urls.table(database.name, table.name) }}"{% if table.count %} title="{{ table.count }} rows"{% endif %}>{{ table.name }}</a>{% if table.private %} 🔒{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_and_views_more %}, <a href="{{ urls.database(database.name) }}">...</a>{% endif %}</p>

datasette/templates/table.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 
-{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
+{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
 
 {% block extra_head %}
 {{- super() -}}
@@ -47,11 +47,12 @@
 {% endif %}
 
 {% if count or human_description_en %}
-    <h3>
+    <h3 class="table-summary">
-        {% if count_truncated %}&gt;{{ "{:,}".format(count - 1) }} rows
+        {% if count_truncated %}<span class="table-count" aria-live="polite">{{ "{:,}".format(count - 1) }}+ rows</span>
-        {% if allow_execute_sql and query.sql %} <a class="count-sql" style="font-size: 0.8em;" href="{{ urls.database_query(database, count_sql) }}">count all</a>{% endif %}
+        <button type="button" class="count-all" data-count-url="{{ urls.table(database, table) }}/-/count">count all</button>
+        <span class="count-error" role="alert"></span>
         {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}
-        {% if human_description_en %}{{ human_description_en }}{% endif %}
+        {% if human_description_en %}<span class="table-summary-description">{{ human_description_en }}</span>{% endif %}
     </h3>
 {% endif %}
 

datasette/tokens.py

@@ -10,7 +10,7 @@ from __future__ import annotations
 
 import dataclasses
 import time
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 import itsdangerous
 
@@ -50,24 +50,24 @@ class TokenRestrictions:
     database: dict[str, list[str]] = dataclasses.field(default_factory=dict)
     resource: dict[str, dict[str, list[str]]] = dataclasses.field(default_factory=dict)
 
-    def allow_all(self, action: str) -> "TokenRestrictions":
+    def allow_all(self, action: str) -> TokenRestrictions:
         """Allow an action across all databases and resources."""
         self.all.append(action)
         return self
 
-    def allow_database(self, database: str, action: str) -> "TokenRestrictions":
+    def allow_database(self, database: str, action: str) -> TokenRestrictions:
         """Allow an action on a specific database."""
         self.database.setdefault(database, []).append(action)
         return self
 
     def allow_resource(
         self, database: str, resource: str, action: str
-    ) -> "TokenRestrictions":
+    ) -> TokenRestrictions:
         """Allow an action on a specific resource within a database."""
         self.resource.setdefault(database, {}).setdefault(resource, []).append(action)
         return self
 
-    def abbreviated(self, datasette: "Datasette") -> Optional[dict]:
+    def abbreviated(self, datasette: Datasette) -> dict | None:
         """
         Return the abbreviated ``_r`` dictionary shape for this set of
         restrictions, using action abbreviations registered with ``datasette``.
@@ -112,16 +112,16 @@ class TokenHandler:
 
     async def create_token(
         self,
-        datasette: "Datasette",
+        datasette: Datasette,
         actor_id: str,
         *,
-        expires_after: Optional[int] = None,
+        expires_after: int | None = None,
-        restrictions: Optional[TokenRestrictions] = None,
+        restrictions: TokenRestrictions | None = None,
     ) -> str:
         """Create and return a token string for the given actor."""
         raise NotImplementedError
 
-    async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]:
+    async def verify_token(self, datasette: Datasette, token: str) -> dict | None:
         """
         Verify a token and return an actor dict.
 
@@ -142,11 +142,11 @@ class SignedTokenHandler(TokenHandler):
 
     async def create_token(
         self,
-        datasette: "Datasette",
+        datasette: Datasette,
         actor_id: str,
         *,
-        expires_after: Optional[int] = None,
+        expires_after: int | None = None,
-        restrictions: Optional[TokenRestrictions] = None,
+        restrictions: TokenRestrictions | None = None,
     ) -> str:
         if not datasette.setting("allow_signed_tokens"):
             raise ValueError(
@@ -163,7 +163,7 @@ class SignedTokenHandler(TokenHandler):
                 token["_r"] = abbreviated
         return "dstok_{}".format(datasette.sign(token, namespace="token"))
 
-    async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]:
+    async def verify_token(self, datasette: Datasette, token: str) -> dict | None:
         prefix = "dstok_"
 
         if not token.startswith(prefix):
@@ -200,9 +200,8 @@ class SignedTokenHandler(TokenHandler):
         ):
             duration = max_signed_tokens_ttl
 
-        if duration:
+        if duration and time.time() - created > duration:
-            if time.time() - created > duration:
+            raise TokenInvalid("Token has expired")
-                raise TokenInvalid("Token has expired")
 
         actor = {"id": decoded["a"], "token": "dstok"}
 

datasette/tracer.py

@@ -1,10 +1,11 @@
 import asyncio
+import json
+import time
+import traceback
 from contextlib import contextmanager
 from contextvars import ContextVar
+
 from markupsafe import escape
-import time
-import json
-import traceback
 
 tracers = {}
 
@@ -132,17 +133,17 @@ class AsgiTracer:
                     "num_traces": len(traces),
                     "traces": traces,
                 }
-                try:
+                content_type = next(
-                    content_type = [
+                    (
                         v.decode("utf8")
                         for k, v in response_headers
                         if k.lower() == b"content-type"
-                    ][0]
+                    ),
-                except IndexError:
+                    "",
-                    content_type = ""
+                )
                 if "text/html" in content_type and b"</body>" in accumulated_body:
                     extra = escape(json.dumps(trace_info, indent=2))
-                    extra_html = f"<pre>{extra}</pre></body>".encode("utf8")
+                    extra_html = f"<pre>{extra}</pre></body>".encode()
                     accumulated_body = accumulated_body.replace(b"</body>", extra_html)
                 elif "json" in content_type and accumulated_body.startswith(b"{"):
                     data = json.loads(accumulated_body.decode("utf8"))

datasette/url_builder.py

@@ -1,6 +1,7 @@
-from .utils import tilde_encode, path_with_format, PrefixedUrlString
 import urllib
 
+from .utils import PrefixedUrlString, path_with_format, tilde_encode
+
 
 class Urls:
     def __init__(self, ds):
@@ -8,8 +9,7 @@ class Urls:
 
     def path(self, path, format=None):
         if not isinstance(path, PrefixedUrlString):
-            if path.startswith("/"):
+            path = path.removeprefix("/")
-                path = path[1:]
             path = self.ds.setting("base_url") + path
         if format is not None:
             path = path_with_format(path=path, format=format)
@@ -56,6 +56,7 @@ class Urls:
         return PrefixedUrlString(path)
 
     def row_blob(self, database, table, row_path, column):
-        return self.table(database, table) + "/{}.blob?_blob_column={}".format(
+        return (
-            row_path, urllib.parse.quote_plus(column)
+            self.table(database, table)
+            + f"/{row_path}.blob?_blob_column={urllib.parse.quote_plus(column)}"
         )

datasette/utils/__init__.py

@@ -1,29 +1,31 @@
 import asyncio
+import base64
 import binascii
-from contextlib import contextmanager
-import aiofiles
-import click
-from collections import OrderedDict, namedtuple, Counter
 import copy
 import dataclasses
-import base64
 import hashlib
 import inspect
 import json
-import markupsafe
-import mergedeep
 import os
 import re
+import secrets
 import shlex
+import shutil
 import tempfile
-import typing
 import time
 import types
-import secrets
+import typing
-import shutil
-from typing import Iterable, List, Tuple
 import urllib
+from collections import Counter, OrderedDict, namedtuple
+from collections.abc import Iterable
+from contextlib import contextmanager
+
+import aiofiles
+import click
+import markupsafe
+import mergedeep
 import yaml
+
 from .shutil_backport import copytree
 from .sqlite import sqlite3, supports_table_xinfo
 
@@ -36,7 +38,7 @@ if typing.TYPE_CHECKING:
 class PaginatedResources:
     """Paginated results from allowed_resources query."""
 
-    resources: List["Resource"]
+    resources: list["Resource"]
     next: str | None  # Keyset token for next page (None if no more results)
     _datasette: typing.Any = dataclasses.field(default=None, repr=False)
     _action: str = dataclasses.field(default=None, repr=False)
@@ -83,22 +85,132 @@ class PaginatedResources:
 
 
 # From https://www.sqlite.org/lang_keywords.html
-reserved_words = set(
+reserved_words = {
-    (
+    "abort",
-        "abort action add after all alter analyze and as asc attach autoincrement "
+    "action",
-        "before begin between by cascade case cast check collate column commit "
+    "add",
-        "conflict constraint create cross current_date current_time "
+    "after",
-        "current_timestamp database default deferrable deferred delete desc detach "
+    "all",
-        "distinct drop each else end escape except exclusive exists explain fail "
+    "alter",
-        "for foreign from full glob group having if ignore immediate in index "
+    "analyze",
-        "indexed initially inner insert instead intersect into is isnull join key "
+    "and",
-        "left like limit match natural no not notnull null of offset on or order "
+    "as",
-        "outer plan pragma primary query raise recursive references regexp reindex "
+    "asc",
-        "release rename replace restrict right rollback row savepoint select set "
+    "attach",
-        "table temp temporary then to transaction trigger union unique update using "
+    "autoincrement",
-        "vacuum values view virtual when where with without"
+    "before",
-    ).split()
+    "begin",
-)
+    "between",
+    "by",
+    "cascade",
+    "case",
+    "cast",
+    "check",
+    "collate",
+    "column",
+    "commit",
+    "conflict",
+    "constraint",
+    "create",
+    "cross",
+    "current_date",
+    "current_time",
+    "current_timestamp",
+    "database",
+    "default",
+    "deferrable",
+    "deferred",
+    "delete",
+    "desc",
+    "detach",
+    "distinct",
+    "drop",
+    "each",
+    "else",
+    "end",
+    "escape",
+    "except",
+    "exclusive",
+    "exists",
+    "explain",
+    "fail",
+    "for",
+    "foreign",
+    "from",
+    "full",
+    "glob",
+    "group",
+    "having",
+    "if",
+    "ignore",
+    "immediate",
+    "in",
+    "index",
+    "indexed",
+    "initially",
+    "inner",
+    "insert",
+    "instead",
+    "intersect",
+    "into",
+    "is",
+    "isnull",
+    "join",
+    "key",
+    "left",
+    "like",
+    "limit",
+    "match",
+    "natural",
+    "no",
+    "not",
+    "notnull",
+    "null",
+    "of",
+    "offset",
+    "on",
+    "or",
+    "order",
+    "outer",
+    "plan",
+    "pragma",
+    "primary",
+    "query",
+    "raise",
+    "recursive",
+    "references",
+    "regexp",
+    "reindex",
+    "release",
+    "rename",
+    "replace",
+    "restrict",
+    "right",
+    "rollback",
+    "row",
+    "savepoint",
+    "select",
+    "set",
+    "table",
+    "temp",
+    "temporary",
+    "then",
+    "to",
+    "transaction",
+    "trigger",
+    "union",
+    "unique",
+    "update",
+    "using",
+    "vacuum",
+    "values",
+    "view",
+    "virtual",
+    "when",
+    "where",
+    "with",
+    "without",
+}
 
 APT_GET_DOCKERFILE_EXTRAS = r"""
 RUN apt-get update && \
@@ -158,7 +270,7 @@ functions_marked_as_documented = []
 
 def documented(fn=None, *, label=None):
     def decorate(fn):
-        fn._datasette_docs_label = label or "internals_utils_{}".format(fn.__name__)
+        fn._datasette_docs_label = label or f"internals_utils_{fn.__name__}"
         functions_marked_as_documented.append(fn)
         return fn
 
@@ -360,7 +472,7 @@ disallawed_sql_res = [
     (
         re.compile(f"pragma(?!_({'|'.join(allowed_pragmas)}))"),
         "Statement contained a disallowed PRAGMA. Allowed pragma functions are {}".format(
-            ", ".join("pragma_{}()".format(pragma) for pragma in allowed_pragmas)
+            ", ".join(f"pragma_{pragma}()" for pragma in allowed_pragmas)
         ),
     )
 ]
@@ -456,7 +568,7 @@ def escape_css_string(s):
 
 
 def escape_sqlite(s):
-    if _boring_keyword_re.match(s) and (s.lower() not in reserved_words):
+    if _boring_keyword_re.fullmatch(s) and (s.lower() not in reserved_words):
         return s
     return '"{}"'.format(s.replace('"', '""'))
 
@@ -534,10 +646,7 @@ CMD {cmd}""".format(
             else ""
         ),
         environment_variables="\n".join(
-            [
+            [f"ENV {key} '{value}'" for key, value in environment_variables.items()]
-                "ENV {} '{}'".format(key, value)
-                for key, value in environment_variables.items()
-            ]
         ),
         install_from=" ".join(install),
         files=" ".join(files),
@@ -636,11 +745,11 @@ def detect_primary_keys(conn, table):
 
 
 def get_outbound_foreign_keys(conn, table):
-    infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall()
+    infos = conn.execute(f"PRAGMA foreign_key_list({escape_sqlite(table)})").fetchall()
     fks = []
     for info in infos:
         if info is not None:
-            id, seq, table_name, from_, to_, on_update, on_delete, match = info
+            id, seq, table_name, from_, to_, _on_update, _on_delete, _match = info
             fks.append(
                 {
                     "column": from_,
@@ -711,7 +820,8 @@ def detect_spatialite(conn):
 
 def detect_fts(conn, table):
     """Detect if table has a corresponding FTS virtual table and return it"""
-    rows = conn.execute(detect_fts_sql(table)).fetchall()
+    sql, params = detect_fts_sql(table)
+    rows = conn.execute(sql, params).fetchall()
     if len(rows) == 0:
         return None
     else:
@@ -719,18 +829,26 @@ def detect_fts(conn, table):
 
 
 def detect_fts_sql(table):
-    return r"""
+    escaped_table = table.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
-        select name from sqlite_master
+    return (
-            where rootpage = 0
+        r"""
-            and (
+            select name from sqlite_master
-                sql like '%VIRTUAL TABLE%USING FTS%content="{table}"%'
+                where rootpage = 0
-                or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%'
+                and (
-                or (
+                    sql like :fts_double_quoted escape char(92)
-                    tbl_name = "{table}"
+                    or sql like :fts_bracket_quoted escape char(92)
-                    and sql like '%VIRTUAL TABLE%USING FTS%'
+                    or (
+                        tbl_name = :table
+                        and sql like '%VIRTUAL TABLE%USING FTS%'
+                    )
                 )
-            )
+        """,
-    """.format(table=table.replace("'", "''"))
+        {
+            "fts_double_quoted": f'%VIRTUAL TABLE%USING FTS%content="{escaped_table}"%',
+            "fts_bracket_quoted": f"%VIRTUAL TABLE%USING FTS%content=[{escaped_table}]%",
+            "table": table,
+        },
+    )
 
 
 def detect_json1(conn=None):
@@ -741,7 +859,7 @@ def detect_json1(conn=None):
     try:
         conn.execute("SELECT json('{}')")
         return True
-    except Exception:
+    except sqlite3.Error:
         return False
     finally:
         if close_conn:
@@ -821,9 +939,7 @@ def is_url(value):
     if not value.startswith("http://") and not value.startswith("https://"):
         return False
     # Any whitespace at all is invalid
-    if whitespace_re.search(value):
+    return not whitespace_re.search(value)
-        return False
-    return True
 
 
 css_class_re = re.compile(r"^[a-zA-Z]+[_a-zA-Z0-9-]*$")
@@ -876,7 +992,9 @@ def module_from_path(path, name):
     mod.__file__ = path
     with open(path, "r") as file:
         code = compile(file.read(), path, "exec", dont_inherit=True)
-    exec(code, mod.__dict__)
+    # Executing the file is the whole point - this is how --plugins-dir loads
+    # plugins and how metadata/config .py files are evaluated
+    exec(code, mod.__dict__)  # noqa: S102
     return mod
 
 
@@ -1033,9 +1151,7 @@ def escape_fts(query):
         query += '"'
     bits = _escape_fts_re.split(query)
     bits = [b for b in bits if b and b != '""']
-    return " ".join(
+    return " ".join(f'"{bit}"' if not bit.startswith('"') else bit for bit in bits)
-        '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits
-    )
 
 
 class MultiParams:
@@ -1047,7 +1163,7 @@ class MultiParams:
                     data[key], (list, tuple)
                 ), "dictionary data should be a dictionary of key => [list]"
             self._data = data
-        elif isinstance(data, list) or isinstance(data, tuple):
+        elif isinstance(data, (list, tuple)):
             new_data = {}
             for item in data:
                 assert (
@@ -1137,9 +1253,7 @@ def _gather_arguments(fn, kwargs):
     for parameter in parameters:
         if parameter not in kwargs:
             raise TypeError(
-                "{} requires parameters {}, missing: {}".format(
+                f"{fn} requires parameters {tuple(parameters)}, missing: {set(parameters) - set(kwargs.keys())}"
-                    fn, tuple(parameters), set(parameters) - set(kwargs.keys())
-                )
             )
         call_with.append(kwargs[parameter])
     return call_with
@@ -1208,9 +1322,9 @@ def resolve_env_secrets(config, environ):
     """Create copy that recursively replaces {"$env": "NAME"} with values from environ"""
     if isinstance(config, dict):
         if list(config.keys()) == ["$env"]:
-            return environ.get(list(config.values())[0])
+            return environ.get(next(iter(config.values())))
         elif list(config.keys()) == ["$file"]:
-            with open(list(config.values())[0]) as fp:
+            with open(next(iter(config.values()))) as fp:
                 return fp.read()
         else:
             return {
@@ -1306,7 +1420,7 @@ _named_param_re = re.compile(r":(\w+)")
 
 
 @documented
-def named_parameters(sql: str) -> List[str]:
+def named_parameters(sql: str) -> list[str]:
     """
     Given a SQL statement, return a list of named parameters that are used in the statement
 
@@ -1319,7 +1433,7 @@ def named_parameters(sql: str) -> List[str]:
     return _named_param_re.findall(sql)
 
 
-async def derive_named_parameters(db: "Database", sql: str) -> List[str]:
+async def derive_named_parameters(db: "Database", sql: str) -> list[str]:
     """
     This undocumented but stable method exists for backwards compatibility
     with plugins that were using it before it switched to named_parameters()
@@ -1343,9 +1457,9 @@ def parse_size_limit(value, default, maximum, name="_size"):
         if size < 0:
             raise ValueError
     except ValueError:
-        raise ValueError("{} must be a positive integer".format(name))
+        raise ValueError(f"{name} must be a positive integer")
     if size > maximum:
-        raise ValueError("{} must be <= {}".format(name, maximum))
+        raise ValueError(f"{name} must be <= {maximum}")
     return size
 
 
@@ -1403,7 +1517,7 @@ class TildeEncoder(dict):
         elif b == _space:
             res = "+"
         else:
-            res = "~{:02X}".format(b)
+            res = f"~{b:02X}"
         self[b] = res
         return res
 
@@ -1452,7 +1566,13 @@ async def row_sql_params_pks(db, table, pk_values):
     if use_rowid:
         select = "rowid, *"
         pks = ["rowid"]
-    wheres = [f'"{pk}"=:p{i}' for i, pk in enumerate(pks)]
+    wheres = []
+    for i, pk in enumerate(pks):
+        escaped_pk = escape_sqlite(pk)
+        # Preserve the historic always-quoted SQL exposed by _extra=query
+        if escaped_pk == pk:
+            escaped_pk = f'"{pk}"'
+        wheres.append(f"{escaped_pk}=:p{i}")
     sql = f"select {select} from {escape_sqlite(table)} where {' AND '.join(wheres)}"
     params = {}
     for i, pk_value in enumerate(pk_values):
@@ -1498,7 +1618,7 @@ def _combine(base: dict, update: dict) -> dict:
     return base
 
 
-def pairs_to_nested_config(pairs: typing.List[typing.Tuple[str, typing.Any]]) -> dict:
+def pairs_to_nested_config(pairs: list[tuple[str, typing.Any]]) -> dict:
     """
     Parse a list of key-value pairs into a nested dictionary.
     """
@@ -1513,7 +1633,7 @@ def make_slot_function(name, datasette, request, **kwargs):
     from datasette.plugins import pm
 
     method = getattr(pm.hook, name, None)
-    assert method is not None, "No hook found for {}".format(name)
+    assert method is not None, f"No hook found for {name}"
 
     async def inner():
         html_bits = []
@@ -1537,7 +1657,7 @@ def prune_empty_dicts(d: dict):
                 d.pop(key, None)
 
 
-def move_plugins_and_allow(source: dict, destination: dict) -> Tuple[dict, dict]:
+def move_plugins_and_allow(source: dict, destination: dict) -> tuple[dict, dict]:
     """
     Move 'plugins' and 'allow' keys from source to destination dictionary. Creates
     hierarchy in destination if needed. After moving, recursively remove any keys
@@ -1624,7 +1744,7 @@ def redact_keys(original: dict, key_patterns: Iterable) -> dict:
             return {
                 k: (
                     redact(v)
-                    if not any(pattern in k for pattern in key_patterns)
+                    if not any(pattern in k.casefold() for pattern in key_patterns)
                     else "***"
                 )
                 for k, v in data.items()

datasette/utils/actions_sql.py

@@ -29,6 +29,15 @@ from datasette.utils.permissions import gather_permission_sql_from_hooks
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
+    from datasette.permissions import Action
+
+
+def _child_collation(action: "Action") -> str:
+    """Match resource identity without changing the spelling returned by SQL."""
+    resource_class = action.resource_class
+    if resource_class is not None and resource_class.case_insensitive_child:
+        return "NOCASE"
+    return "BINARY"
 
 
 async def build_allowed_resources_sql(
@@ -149,6 +158,7 @@ async def _build_single_action_sql(
         raise ValueError(f"Unknown action: {action}")
 
     # Get base resources SQL from the resource class
+    child_collation = _child_collation(action_obj)
     base_resources_sql = await action_obj.resource_class.resources_sql(
         datasette, actor=actor
     )
@@ -185,7 +195,7 @@ async def _build_single_action_sql(
         if permission_sql.sql is None:
             continue
         rule_sqls.append(f"""
-            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
+            SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
             """.strip())
@@ -252,88 +262,62 @@ async def _build_single_action_sql(
                 ]
             )
 
-    # Continue with the cascading logic
+    # Continue with the cascading logic.
-    query_parts.extend(
+    # Aggregate the RULES by cascade level (small), rather than grouping
-        [
+    # base x rules (which scales with the number of resources).
-            "child_lvl AS (",
+    def _agg(select_key, where, group_by):
-            "  SELECT b.parent, b.child,",
+        parts = [
-            "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
+            f"  SELECT {select_key}",
-            "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,",
+            "         MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-            "         json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,",
+            "         MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow,",
-            "         json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons",
+            "         json_group_array(CASE WHEN allow = 0 THEN source_plugin || ': ' || reason END) AS deny_reasons,",
-            "  FROM base b",
+            "         json_group_array(CASE WHEN allow = 1 THEN source_plugin || ': ' || reason END) AS allow_reasons",
-            "  LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child = b.child",
+            f"  FROM all_rules WHERE {where}",
-            "  GROUP BY b.parent, b.child",
-            "),",
-            "parent_lvl AS (",
-            "  SELECT b.parent, b.child,",
-            "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-            "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,",
-            "         json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,",
-            "         json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons",
-            "  FROM base b",
-            "  LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child IS NULL",
-            "  GROUP BY b.parent, b.child",
-            "),",
-            "global_lvl AS (",
-            "  SELECT b.parent, b.child,",
-            "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-            "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,",
-            "         json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,",
-            "         json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons",
-            "  FROM base b",
-            "  LEFT JOIN all_rules ar ON ar.parent IS NULL AND ar.child IS NULL",
-            "  GROUP BY b.parent, b.child",
-            "),",
         ]
+        if group_by:
+            parts.append(f"  GROUP BY {group_by}")
+        return parts
+
+    query_parts.extend(
+        ["child_agg AS ("]
+        + _agg(
+            "parent, child,",
+            "parent IS NOT NULL AND child IS NOT NULL",
+            "parent, child",
+        )
+        + ["),", "parent_agg AS ("]
+        + _agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent")
+        + ["),", "global_agg AS ("]
+        + _agg("", "parent IS NULL AND child IS NULL", None)
+        + ["),"]
     )
 
     # Add anonymous decision logic if needed
     if include_is_private:
-        query_parts.extend(
+
-            [
+        def _anon_agg(select_key, where, group_by):
-                "anon_child_lvl AS (",
+            parts = [
-                "  SELECT b.parent, b.child,",
+                f"  SELECT {select_key}",
-                "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
+                "         MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-                "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow",
+                "         MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow",
-                "  FROM base b",
+                f"  FROM anon_rules WHERE {where}",
-                "  LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child = b.child",
-                "  GROUP BY b.parent, b.child",
-                "),",
-                "anon_parent_lvl AS (",
-                "  SELECT b.parent, b.child,",
-                "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-                "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow",
-                "  FROM base b",
-                "  LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child IS NULL",
-                "  GROUP BY b.parent, b.child",
-                "),",
-                "anon_global_lvl AS (",
-                "  SELECT b.parent, b.child,",
-                "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-                "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow",
-                "  FROM base b",
-                "  LEFT JOIN anon_rules ar ON ar.parent IS NULL AND ar.child IS NULL",
-                "  GROUP BY b.parent, b.child",
-                "),",
-                "anon_decisions AS (",
-                "  SELECT",
-                "    b.parent, b.child,",
-                "    CASE",
-                "      WHEN acl.any_deny = 1 THEN 0",
-                "      WHEN acl.any_allow = 1 THEN 1",
-                "      WHEN apl.any_deny = 1 THEN 0",
-                "      WHEN apl.any_allow = 1 THEN 1",
-                "      WHEN agl.any_deny = 1 THEN 0",
-                "      WHEN agl.any_allow = 1 THEN 1",
-                "      ELSE 0",
-                "    END AS anon_is_allowed",
-                "  FROM base b",
-                "  JOIN anon_child_lvl acl ON b.parent = acl.parent AND (b.child = acl.child OR (b.child IS NULL AND acl.child IS NULL))",
-                "  JOIN anon_parent_lvl apl ON b.parent = apl.parent AND (b.child = apl.child OR (b.child IS NULL AND apl.child IS NULL))",
-                "  JOIN anon_global_lvl agl ON b.parent = agl.parent AND (b.child = agl.child OR (b.child IS NULL AND agl.child IS NULL))",
-                "),",
             ]
+            if group_by:
+                parts.append(f"  GROUP BY {group_by}")
+            return parts
+
+        query_parts.extend(
+            ["anon_child_agg AS ("]
+            + _anon_agg(
+                f"parent, child COLLATE {child_collation} AS child,",
+                "parent IS NOT NULL AND child IS NOT NULL",
+                f"parent, child COLLATE {child_collation}",
+            )
+            + ["),", "anon_parent_agg AS ("]
+            + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent")
+            + ["),", "anon_global_agg AS ("]
+            + _anon_agg("", "parent IS NULL AND child IS NULL", None)
+            + ["),"]
         )
 
     # Final decisions
@@ -342,31 +326,28 @@ async def _build_single_action_sql(
             "decisions AS (",
             "  SELECT",
             "    b.parent, b.child,",
-            "    -- Cascading permission logic: child → parent → global, DENY beats ALLOW at each level",
+            "    -- Cascading permission logic: child -> parent -> global, DENY beats ALLOW at each level",
             "    -- Priority order:",
-            "    --   1. Child-level deny (most specific, blocks access)",
+            "    --   1. Child-level deny  2. Child-level allow",
-            "    --   2. Child-level allow (most specific, grants access)",
+            "    --   3. Parent-level deny 4. Parent-level allow",
-            "    --   3. Parent-level deny (intermediate, blocks access)",
+            "    --   5. Global-level deny 6. Global-level allow",
-            "    --   4. Parent-level allow (intermediate, grants access)",
-            "    --   5. Global-level deny (least specific, blocks access)",
-            "    --   6. Global-level allow (least specific, grants access)",
             "    --   7. Default deny (no rules match)",
             "    CASE",
-            "      WHEN cl.any_deny = 1 THEN 0",
+            "      WHEN ca.any_deny = 1 THEN 0",
-            "      WHEN cl.any_allow = 1 THEN 1",
+            "      WHEN ca.any_allow = 1 THEN 1",
-            "      WHEN pl.any_deny = 1 THEN 0",
+            "      WHEN pa.any_deny = 1 THEN 0",
-            "      WHEN pl.any_allow = 1 THEN 1",
+            "      WHEN pa.any_allow = 1 THEN 1",
-            "      WHEN gl.any_deny = 1 THEN 0",
+            "      WHEN ga.any_deny = 1 THEN 0",
-            "      WHEN gl.any_allow = 1 THEN 1",
+            "      WHEN ga.any_allow = 1 THEN 1",
             "      ELSE 0",
             "    END AS is_allowed,",
             "    CASE",
-            "      WHEN cl.any_deny = 1 THEN cl.deny_reasons",
+            "      WHEN ca.any_deny = 1 THEN ca.deny_reasons",
-            "      WHEN cl.any_allow = 1 THEN cl.allow_reasons",
+            "      WHEN ca.any_allow = 1 THEN ca.allow_reasons",
-            "      WHEN pl.any_deny = 1 THEN pl.deny_reasons",
+            "      WHEN pa.any_deny = 1 THEN pa.deny_reasons",
-            "      WHEN pl.any_allow = 1 THEN pl.allow_reasons",
+            "      WHEN pa.any_allow = 1 THEN pa.allow_reasons",
-            "      WHEN gl.any_deny = 1 THEN gl.deny_reasons",
+            "      WHEN ga.any_deny = 1 THEN ga.deny_reasons",
-            "      WHEN gl.any_allow = 1 THEN gl.allow_reasons",
+            "      WHEN ga.any_allow = 1 THEN ga.allow_reasons",
             "      ELSE '[]'",
             "    END AS reason",
         ]
@@ -374,21 +355,34 @@ async def _build_single_action_sql(
 
     if include_is_private:
         query_parts.append(
-            "    , CASE WHEN ad.anon_is_allowed = 0 THEN 1 ELSE 0 END AS is_private"
+            "    , CASE WHEN ("
+            "CASE"
+            " WHEN aca.any_deny = 1 THEN 0"
+            " WHEN aca.any_allow = 1 THEN 1"
+            " WHEN apa.any_deny = 1 THEN 0"
+            " WHEN apa.any_allow = 1 THEN 1"
+            " WHEN aga.any_deny = 1 THEN 0"
+            " WHEN aga.any_allow = 1 THEN 1"
+            " ELSE 0 END"
+            ") = 0 THEN 1 ELSE 0 END AS is_private"
         )
 
     query_parts.extend(
         [
             "  FROM base b",
-            "  JOIN child_lvl cl ON b.parent = cl.parent AND (b.child = cl.child OR (b.child IS NULL AND cl.child IS NULL))",
+            "  LEFT JOIN child_agg ca ON ca.parent = b.parent AND ca.child = b.child",
-            "  JOIN parent_lvl pl ON b.parent = pl.parent AND (b.child = pl.child OR (b.child IS NULL AND pl.child IS NULL))",
+            "  LEFT JOIN parent_agg pa ON pa.parent = b.parent",
-            "  JOIN global_lvl gl ON b.parent = gl.parent AND (b.child = gl.child OR (b.child IS NULL AND gl.child IS NULL))",
+            "  CROSS JOIN global_agg ga",
         ]
     )
 
     if include_is_private:
-        query_parts.append(
+        query_parts.extend(
-            "  JOIN anon_decisions ad ON b.parent = ad.parent AND (b.child = ad.child OR (b.child IS NULL AND ad.child IS NULL))"
+            [
+                "  LEFT JOIN anon_child_agg aca ON aca.parent = b.parent AND aca.child = b.child",
+                "  LEFT JOIN anon_parent_agg apa ON apa.parent = b.parent",
+                "  CROSS JOIN anon_global_agg aga",
+            ]
         )
 
     query_parts.append(")")
@@ -398,10 +392,31 @@ async def _build_single_action_sql(
         # Wrap each restriction_sql in a subquery to avoid operator precedence issues
         # with UNION ALL inside the restriction SQL statements
         restriction_intersect = "\nINTERSECT\n".join(
-            f"SELECT * FROM ({sql})" for sql in restriction_sqls
+            f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})"
+            for sql in restriction_sqls
         )
+        # Decompose by NULL-pattern so the final filter can use pure-equality
+        # EXISTS lookups (satisfiable via automatic indexes) instead of a
+        # correlated OR-scan over the whole list.
         query_parts.extend(
-            [",", "restriction_list AS (", f"  {restriction_intersect}", ")"]
+            [
+                ",",
+                "restriction_list AS (",
+                f"  {restriction_intersect}",
+                "),",
+                "restriction_exact AS (",
+                "  SELECT parent, child FROM restriction_list WHERE parent IS NOT NULL AND child IS NOT NULL",
+                "),",
+                "restriction_parent_any AS (",
+                "  SELECT DISTINCT parent FROM restriction_list WHERE parent IS NOT NULL AND child IS NULL",
+                "),",
+                "restriction_child_any AS (",
+                "  SELECT DISTINCT child FROM restriction_list WHERE parent IS NULL AND child IS NOT NULL",
+                "),",
+                "restriction_all AS (",
+                "  SELECT 1 AS matched FROM restriction_list WHERE parent IS NULL AND child IS NULL LIMIT 1",
+                ")",
+            ]
         )
 
     # Final SELECT
@@ -416,10 +431,11 @@ async def _build_single_action_sql(
     # Add restriction filter if there are restrictions
     if restriction_sqls:
         query_parts.append("""
-  AND EXISTS (
+  AND (
-    SELECT 1 FROM restriction_list r
+    EXISTS (SELECT 1 FROM restriction_all)
-    WHERE (r.parent = decisions.parent OR r.parent IS NULL)
+    OR EXISTS (SELECT 1 FROM restriction_parent_any r WHERE r.parent = decisions.parent)
-      AND (r.child = decisions.child OR r.child IS NULL)
+    OR EXISTS (SELECT 1 FROM restriction_child_any r WHERE r.child = decisions.child)
+    OR EXISTS (SELECT 1 FROM restriction_exact r WHERE r.parent = decisions.parent AND r.child = decisions.child)
   )""")
 
     # Add parent filter if specified
@@ -475,6 +491,7 @@ async def build_permission_rules_sql(
     union_parts = []
     all_params = {}
     restriction_sqls = []
+    child_collation = _child_collation(action_obj)
 
     for permission_sql in permission_sqls:
         all_params.update(permission_sql.params or {})
@@ -488,7 +505,7 @@ async def build_permission_rules_sql(
             continue
 
         union_parts.append(f"""
-            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
+            SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
             """.strip())
@@ -559,6 +576,7 @@ async def check_permissions_for_actions(
     verdicts = {}
 
     for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)):
+        child_collation = _child_collation(datasette.actions[action])
         prefix = f"a{i}_"
         rule_parts = []
         restriction_parts = []
@@ -584,7 +602,7 @@ async def check_permissions_for_actions(
             if sql is None:
                 continue
             rule_parts.append(
-                f"SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
+                f"SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
             )
 
         if not rule_parts:
@@ -618,7 +636,8 @@ async def check_permissions_for_actions(
         if restriction_parts:
             # Database-level restrictions (parent, NULL) match all children
             restriction_intersect = "\nINTERSECT\n".join(
-                f"SELECT * FROM ({sql})" for sql in restriction_parts
+                f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})"
+                for sql in restriction_parts
             )
             ctes.append(f"a{i}_restriction AS (\n{restriction_intersect}\n)")
             verdict_sql = f"""({verdict_sql}) AND EXISTS (
@@ -673,3 +692,240 @@ async def check_permission_for_resource(
         child=child,
     )
     return results[action]
+
+
+async def explain_permission_for_resource(
+    *,
+    datasette: "Datasette",
+    actor: dict | None,
+    action: str,
+    parent: str | None,
+    child: str | None,
+) -> dict:
+    """Explain a permission decision for one action and resource.
+
+    This is intended for Datasette's permission debugging tools. It uses the
+    same ``permission_resources_sql`` hook results and the same resolution
+    rules as :func:`check_permissions_for_actions`, but also returns the
+    matching rules, actor restriction results and ``also_requires`` chain.
+
+    The returned dictionary is part of Datasette's unstable debugging API.
+    """
+
+    action_obj = datasette.actions.get(action)
+    if action_obj is None:
+        raise ValueError(f"Unknown action: {action}")
+
+    explanation = await _explain_single_action(
+        datasette=datasette,
+        actor=actor,
+        action=action,
+        parent=parent,
+        child=child,
+    )
+
+    required_actions = []
+    if action_obj.also_requires:
+        required = await explain_permission_for_resource(
+            datasette=datasette,
+            actor=actor,
+            action=action_obj.also_requires,
+            parent=parent,
+            child=child,
+        )
+        required_actions.append(required)
+
+    explanation["required_actions"] = required_actions
+    explanation["allowed"] = bool(
+        explanation["rule_allowed"]
+        and explanation["restriction_allowed"]
+        and all(required["allowed"] for required in required_actions)
+    )
+    explanation["summary"] = _permission_explanation_summary(explanation)
+    return explanation
+
+
+async def _explain_single_action(
+    *,
+    datasette: "Datasette",
+    actor: dict | None,
+    action: str,
+    parent: str | None,
+    child: str | None,
+) -> dict:
+    """Return matching rules and restrictions for a single action."""
+    from datasette.utils.permissions import SKIP_PERMISSION_CHECKS
+
+    permission_sqls = await gather_permission_sql_from_hooks(
+        datasette=datasette,
+        actor=actor,
+        action=action,
+    )
+
+    if permission_sqls is SKIP_PERMISSION_CHECKS:
+        return {
+            "action": action,
+            "rule_allowed": True,
+            "restriction_allowed": True,
+            "winning_scope": "global",
+            "matched_rules": [
+                {
+                    "scope": "global",
+                    "effect": "allow",
+                    "source": "skip_permission_checks",
+                    "reason": "Permission checks were explicitly skipped",
+                    "decisive": True,
+                    "ignored_because": None,
+                }
+            ],
+            "restrictions": [],
+        }
+
+    db = datasette.get_internal_database()
+    matched_rules = []
+    restrictions = []
+    child_collation = _child_collation(datasette.actions[action])
+
+    for permission_sql in permission_sqls:
+        params = dict(permission_sql.params or {})
+        parent_param = _unused_parameter_name(params, "_explain_parent")
+        params[parent_param] = parent
+        child_param = _unused_parameter_name(params, "_explain_child")
+        params[child_param] = child
+
+        if permission_sql.sql:
+            rows = await db.execute(
+                f"""
+                SELECT parent, child, allow, reason
+                FROM ({permission_sql.sql}) AS permission_rules
+                WHERE (parent IS NULL OR parent = :{parent_param})
+                  AND (child IS NULL OR child COLLATE {child_collation} = :{child_param})
+                """,
+                params,
+            )
+            for row in rows:
+                specificity = (
+                    2
+                    if row["child"] is not None
+                    else 1 if row["parent"] is not None else 0
+                )
+                matched_rules.append(
+                    {
+                        "scope": ("resource", "parent", "global")[2 - specificity],
+                        "effect": "allow" if row["allow"] else "deny",
+                        "source": permission_sql.source,
+                        "reason": row["reason"],
+                        "_specificity": specificity,
+                    }
+                )
+
+        if permission_sql.restriction_sql:
+            restriction_row = (
+                await db.execute(
+                    f"""
+                    SELECT EXISTS(
+                        SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules
+                        WHERE (parent IS NULL OR parent = :{parent_param})
+                          AND (child IS NULL OR child COLLATE {child_collation} = :{child_param})
+                    ) AS resource_is_in_allowlist
+                    """,
+                    params,
+                )
+            ).first()
+            restriction_allowed = bool(restriction_row[0])
+            restrictions.append(
+                {
+                    "source": permission_sql.source,
+                    "allowed": restriction_allowed,
+                    "reason": params.get("deny")
+                    or (
+                        "Resource is included in this restriction allowlist"
+                        if restriction_allowed
+                        else "Resource is not included in this restriction allowlist"
+                    ),
+                }
+            )
+
+    matched_rules.sort(
+        key=lambda rule: (
+            -rule["_specificity"],
+            0 if rule["effect"] == "deny" else 1,
+            rule["source"] or "",
+            rule["reason"] or "",
+        )
+    )
+
+    if matched_rules:
+        winning_specificity = matched_rules[0]["_specificity"]
+        winning_rules = [
+            rule
+            for rule in matched_rules
+            if rule["_specificity"] == winning_specificity
+        ]
+        rule_allowed = not any(rule["effect"] == "deny" for rule in winning_rules)
+        winning_scope = winning_rules[0]["scope"]
+    else:
+        winning_specificity = None
+        rule_allowed = False
+        winning_scope = None
+
+    for rule in matched_rules:
+        specificity = rule.pop("_specificity")
+        if specificity != winning_specificity:
+            rule["decisive"] = False
+            rule["ignored_because"] = "A more specific rule matched"
+        elif not rule_allowed and rule["effect"] == "allow":
+            rule["decisive"] = False
+            rule["ignored_because"] = "A deny rule matched at the same scope"
+        else:
+            rule["decisive"] = True
+            rule["ignored_because"] = None
+
+    return {
+        "action": action,
+        "rule_allowed": rule_allowed,
+        "restriction_allowed": all(
+            restriction["allowed"] for restriction in restrictions
+        ),
+        "winning_scope": winning_scope,
+        "matched_rules": matched_rules,
+        "restrictions": restrictions,
+    }
+
+
+def _unused_parameter_name(params: dict, preferred: str) -> str:
+    """Return a SQL parameter name that is not already in ``params``."""
+    candidate = preferred
+    suffix = 2
+    while candidate in params:
+        candidate = f"{preferred}_{suffix}"
+        suffix += 1
+    return candidate
+
+
+def _permission_explanation_summary(explanation: dict) -> str:
+    denied_requirement = next(
+        (
+            required
+            for required in explanation["required_actions"]
+            if not required["allowed"]
+        ),
+        None,
+    )
+    if denied_requirement:
+        return (
+            f"Denied because {explanation['action']} also requires "
+            f"{denied_requirement['action']}, which was denied."
+        )
+    if not explanation["matched_rules"]:
+        return "Denied because no permission rule matched this actor and resource."
+    if not explanation["rule_allowed"]:
+        return (
+            f"Denied by a {explanation['winning_scope']}-level rule. "
+            "Deny rules take precedence over allow rules at the same scope."
+        )
+    if not explanation["restriction_allowed"]:
+        return (
+            "Denied because the resource is not included in the actor's restrictions."
+        )
+    return f"Allowed by the matching {explanation['winning_scope']}-level rule."

datasette/utils/asgi.py

@@ -1,28 +1,30 @@
+import asyncio
 import json
-from typing import Optional
+import re
+from http.cookies import Morsel, SimpleCookie
+from mimetypes import guess_type
+from pathlib import Path
+from urllib.parse import parse_qs, parse_qsl, urlunparse
+
+import aiofiles
+import aiofiles.os
+
 from datasette.utils import MultiParams, calculate_etag, error_body, sha256_file
 from datasette.utils.multipart import (
-    parse_form_data,
-    MultipartParseError,
-    FormData,
-    DEFAULT_MAX_FILE_SIZE,
-    DEFAULT_MAX_REQUEST_SIZE,
-    DEFAULT_MAX_FIELDS,
-    DEFAULT_MAX_FILES,
-    DEFAULT_MAX_PARTS,
     DEFAULT_MAX_FIELD_SIZE,
+    DEFAULT_MAX_FIELDS,
+    DEFAULT_MAX_FILE_SIZE,
+    DEFAULT_MAX_FILES,
     DEFAULT_MAX_MEMORY_FILE_SIZE,
     DEFAULT_MAX_PART_HEADER_BYTES,
     DEFAULT_MAX_PART_HEADER_LINES,
+    DEFAULT_MAX_PARTS,
+    DEFAULT_MAX_REQUEST_SIZE,
     DEFAULT_MIN_FREE_DISK_BYTES,
+    FormData,
+    MultipartParseError,
+    parse_form_data,
 )
-from mimetypes import guess_type
-from urllib.parse import parse_qs, urlunparse, parse_qsl
-from pathlib import Path
-from http.cookies import SimpleCookie, Morsel
-import aiofiles
-import aiofiles.os
-import re
 
 # Workaround for adding samesite support to pre 3.8 python
 Morsel._reserved["samesite"] = "SameSite"
@@ -81,6 +83,19 @@ SAMESITE_VALUES = ("strict", "lax", "none")
 DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024  # 2MB
 
 
+class _RequestHeaders(dict):
+    """Incoming headers with lowercase keys and case-insensitive lookups."""
+
+    def __getitem__(self, key):
+        return super().__getitem__(key.lower())
+
+    def get(self, key, default=None):
+        return super().get(key.lower(), default)
+
+    def __contains__(self, key):
+        return super().__contains__(key.lower())
+
+
 class Request:
     def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES):
         self.scope = scope
@@ -88,7 +103,7 @@ class Request:
         self.max_post_body_bytes = max_post_body_bytes
 
     def __repr__(self):
-        return '<asgi.Request method="{}" url="{}">'.format(self.method, self.url)
+        return f'<asgi.Request method="{self.method}" url="{self.url}">'
 
     @property
     def method(self):
@@ -110,10 +125,10 @@ class Request:
 
     @property
     def headers(self):
-        return {
+        return _RequestHeaders(
-            k.decode("latin-1").lower(): v.decode("latin-1")
+            (k.decode("latin-1").lower(), v.decode("latin-1"))
             for k, v in self.scope.get("headers") or []
-        }
+        )
 
     @property
     def host(self):
@@ -167,7 +182,7 @@ class Request:
         if max_bytes is None:
             max_bytes = self.max_post_body_bytes
         too_large = PayloadTooLarge(
-            "Request body exceeded maximum size of {} bytes".format(max_bytes)
+            f"Request body exceeded maximum size of {max_bytes} bytes"
         )
         if max_bytes:
             # Reject early if the client declares an oversized body
@@ -206,7 +221,7 @@ class Request:
         max_request_size: int = DEFAULT_MAX_REQUEST_SIZE,
         max_fields: int = DEFAULT_MAX_FIELDS,
         max_files: int = DEFAULT_MAX_FILES,
-        max_parts: Optional[int] = DEFAULT_MAX_PARTS,
+        max_parts: int | None = DEFAULT_MAX_PARTS,
         max_field_size: int = DEFAULT_MAX_FIELD_SIZE,
         max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE,
         max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES,
@@ -299,12 +314,24 @@ class AsgiLifespan:
             while True:
                 message = await receive()
                 if message["type"] == "lifespan.startup":
-                    for fn in self.on_startup:
+                    try:
-                        await fn()
+                        for fn in self.on_startup:
+                            await fn()
+                    except Exception as e:  # noqa: BLE001
+                        await send(
+                            {"type": "lifespan.startup.failed", "message": str(e)}
+                        )
+                        return
                     await send({"type": "lifespan.startup.complete"})
                 elif message["type"] == "lifespan.shutdown":
-                    for fn in self.on_shutdown:
+                    try:
-                        await fn()
+                        for fn in self.on_shutdown:
+                            await fn()
+                    except Exception as e:  # noqa: BLE001
+                        await send(
+                            {"type": "lifespan.shutdown.failed", "message": str(e)}
+                        )
+                        return
                     await send({"type": "lifespan.shutdown.complete"})
                     return
         else:
@@ -484,6 +511,8 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None):
             await asgi_send_html(send, "404: File not found", 404)
             return
 
+    # Only the actual static-file handler can bypass dynamic response privacy.
+    inner_static._datasette_static = True
     return inner_static
 
 
@@ -529,9 +558,9 @@ class Response:
         httponly=False,
         samesite="lax",
     ):
-        assert samesite in SAMESITE_VALUES, "samesite should be one of {}".format(
+        assert (
-            SAMESITE_VALUES
+            samesite in SAMESITE_VALUES
-        )
+        ), f"samesite should be one of {SAMESITE_VALUES}"
         cookie = SimpleCookie()
         cookie[key] = value
         for prop_name, prop_value in (
@@ -623,10 +652,23 @@ class AsgiRunOnFirstRequest:
         self.asgi = asgi
         self.on_startup = on_startup
         self._started = False
+        # Guards against concurrent early requests interleaving with startup:
+        # without this, several requests could all observe `_started is
+        # False` and proceed before any of them finish running the hooks.
+        self._lock = asyncio.Lock()
 
     async def __call__(self, scope, receive, send):
-        if not self._started:
+        # Leave "lifespan" scope events alone - this shim only exists as a
-            self._started = True
+        # fallback for hosts that never send them. It wraps AsgiLifespan, so
-            for hook in self.on_startup:
+        # if it ran on_startup here too, a startup exception would escape
-                await hook()
+        # before AsgiLifespan's own try/except got a chance to turn it into
+        # a lifespan.startup.failed message.
+        if scope["type"] != "lifespan" and not self._started:
+            async with self._lock:
+                # Re-check: another request may have finished startup while
+                # we were waiting for the lock.
+                if not self._started:
+                    for hook in self.on_startup:
+                        await hook()
+                    self._started = True
         return await self.asgi(scope, receive, send)

datasette/utils/baseconv.py

@@ -13,7 +13,7 @@ Originally shared here: https://www.djangosnippets.org/snippets/1431/
 """
 
 
-class BaseConverter(object):
+class BaseConverter:
     decimal_digits = "0123456789"
 
     def __init__(self, digits):

datasette/utils/check_callable.py

@@ -1,6 +1,6 @@
 import inspect
 import types
-from typing import NamedTuple, Any
+from typing import Any, NamedTuple
 
 
 class CallableStatus(NamedTuple):
@@ -19,7 +19,7 @@ def check_callable(obj: Any) -> CallableStatus:
     if isinstance(obj, types.FunctionType):
         return CallableStatus(True, inspect.iscoroutinefunction(obj))
 
-    if hasattr(obj, "__call__"):
+    if callable(obj):
         return CallableStatus(True, inspect.iscoroutinefunction(obj.__call__))
 
-    assert False, "obj {} is somehow callable with no __call__ method".format(repr(obj))
+    assert False, f"obj {obj!r} is somehow callable with no __call__ method"

datasette/utils/internal_db.py

@@ -3,7 +3,7 @@ import textwrap
 from sqlite_utils import Database as SQLiteUtilsDatabase
 from sqlite_utils import Migrations
 
-from datasette.utils import table_column_details
+from datasette.utils import escape_sqlite, table_column_details
 
 INTERNAL_DB_SCHEMA_TABLES = {
     "catalog_databases",
@@ -180,29 +180,9 @@ async def init_internal_db(db):
     await db.execute_write_fn(apply_migrations, transaction=False)
 
 
-async def populate_schema_tables(internal_db, db):
+async def populate_schema_tables(internal_db, db, schema_version):
     database_name = db.name
 
-    def delete_everything(conn):
-        conn.execute(
-            "DELETE FROM catalog_tables WHERE database_name = ?", [database_name]
-        )
-        conn.execute(
-            "DELETE FROM catalog_views WHERE database_name = ?", [database_name]
-        )
-        conn.execute(
-            "DELETE FROM catalog_columns WHERE database_name = ?", [database_name]
-        )
-        conn.execute(
-            "DELETE FROM catalog_foreign_keys WHERE database_name = ?",
-            [database_name],
-        )
-        conn.execute(
-            "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name]
-        )
-
-    await internal_db.execute_write_fn(delete_everything)
-
     tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
     views = (await db.execute("select * from sqlite_master WHERE type = 'view'")).rows
 
@@ -227,25 +207,30 @@ async def populate_schema_tables(internal_db, db):
             columns = table_column_details(conn, table_name)
             columns_to_insert.extend(
                 {
-                    **{"database_name": database_name, "table_name": table_name},
+                    "database_name": database_name,
+                    "table_name": table_name,
                     **column._asdict(),
                 }
                 for column in columns
             )
             foreign_keys = conn.execute(
-                f"PRAGMA foreign_key_list([{table_name}])"
+                f"PRAGMA foreign_key_list({escape_sqlite(table_name)})"
             ).fetchall()
             foreign_keys_to_insert.extend(
                 {
-                    **{"database_name": database_name, "table_name": table_name},
+                    "database_name": database_name,
+                    "table_name": table_name,
                     **dict(foreign_key),
                 }
                 for foreign_key in foreign_keys
             )
-            indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall()
+            indexes = conn.execute(
+                f"PRAGMA index_list({escape_sqlite(table_name)})"
+            ).fetchall()
             indexes_to_insert.extend(
                 {
-                    **{"database_name": database_name, "table_name": table_name},
+                    "database_name": database_name,
+                    "table_name": table_name,
                     **dict(index),
                 }
                 for index in indexes
@@ -266,47 +251,76 @@ async def populate_schema_tables(internal_db, db):
         indexes_to_insert,
     ) = await db.execute_fn(collect_info)
 
-    await internal_db.execute_write_many(
+    def replace_catalog(conn):
-        """
+        # Delete child rows before their catalog_tables parents so this also
-        INSERT INTO catalog_tables (database_name, table_name, rootpage, sql)
+        # works if a prepare_connection plugin enables foreign key enforcement.
-        values (?, ?, ?, ?)
+        for table in (
-    """,
+            "catalog_columns",
-        tables_to_insert,
+            "catalog_foreign_keys",
-    )
+            "catalog_indexes",
-    await internal_db.execute_write_many(
+            "catalog_views",
-        """
+            "catalog_tables",
-        INSERT INTO catalog_views (database_name, view_name, rootpage, sql)
+        ):
-        values (?, ?, ?, ?)
+            conn.execute(
-    """,
+                f"DELETE FROM {table} WHERE database_name = ?",
-        views_to_insert,
+                [database_name],
-    )
+            )
-    await internal_db.execute_write_many(
+        conn.execute(
-        """
+            """
-        INSERT INTO catalog_columns (
+            INSERT OR REPLACE INTO catalog_databases (
-            database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
+                database_name, path, is_memory, schema_version
-        ) VALUES (
+            ) VALUES (?, ?, ?, ?)
-            :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
+            """,
+            [
+                database_name,
+                str(db.path) if db.path is not None else None,
+                db.is_memory,
+                schema_version,
+            ],
         )
-    """,
+        conn.executemany(
-        columns_to_insert,
+            """
-    )
+            INSERT INTO catalog_tables (database_name, table_name, rootpage, sql)
-    await internal_db.execute_write_many(
+            values (?, ?, ?, ?)
-        """
+            """,
-        INSERT INTO catalog_foreign_keys (
+            tables_to_insert,
-            database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
-        ) VALUES (
-            :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
         )
-    """,
+        conn.executemany(
-        foreign_keys_to_insert,
+            """
-    )
+            INSERT INTO catalog_views (database_name, view_name, rootpage, sql)
-    await internal_db.execute_write_many(
+            values (?, ?, ?, ?)
-        """
+            """,
-        INSERT INTO catalog_indexes (
+            views_to_insert,
-            database_name, table_name, seq, name, "unique", origin, partial
-        ) VALUES (
-            :database_name, :table_name, :seq, :name, :unique, :origin, :partial
         )
-    """,
+        conn.executemany(
-        indexes_to_insert,
+            """
-    )
+            INSERT INTO catalog_columns (
+                database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
+            ) VALUES (
+                :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
+            )
+            """,
+            columns_to_insert,
+        )
+        conn.executemany(
+            """
+            INSERT INTO catalog_foreign_keys (
+                database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
+            ) VALUES (
+                :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
+            )
+            """,
+            foreign_keys_to_insert,
+        )
+        conn.executemany(
+            """
+            INSERT INTO catalog_indexes (
+                database_name, table_name, seq, name, "unique", origin, partial
+            ) VALUES (
+                :database_name, :table_name, :seq, :name, :unique, :origin, :partial
+            )
+            """,
+            indexes_to_insert,
+        )
+
+    await internal_db.execute_write_fn(replace_catalog)

datasette/utils/multipart.py

@@ -11,15 +11,10 @@ Supports:
 import asyncio
 import shutil
 import tempfile
+from collections.abc import Callable
 from dataclasses import dataclass, field
 from typing import (
     Any,
-    Callable,
-    Dict,
-    List,
-    Optional,
-    Tuple,
-    Union,
 )
 from urllib.parse import parse_qsl
 
@@ -29,7 +24,7 @@ DEFAULT_MAX_REQUEST_SIZE = 100 * 1024 * 1024  # 100MB
 DEFAULT_MAX_FIELDS = 1000
 DEFAULT_MAX_FILES = 100
 # If max_parts is not specified, it defaults to max_fields + max_files
-DEFAULT_MAX_PARTS: Optional[int] = None
+DEFAULT_MAX_PARTS: int | None = None
 DEFAULT_MAX_FIELD_SIZE = 100 * 1024  # 100KB
 DEFAULT_MAX_MEMORY_FILE_SIZE = 1024 * 1024  # 1MB
 DEFAULT_MAX_PART_HEADER_BYTES = 16 * 1024  # 16KB
@@ -40,8 +35,6 @@ DEFAULT_MIN_FREE_DISK_BYTES = 50 * 1024 * 1024  # 50MB
 class MultipartParseError(Exception):
     """Raised when multipart parsing fails."""
 
-    pass
-
 
 @dataclass
 class UploadedFile:
@@ -57,7 +50,7 @@ class UploadedFile:
 
     name: str
     filename: str
-    content_type: Optional[str]
+    content_type: str | None
     size: int
     _file: tempfile.SpooledTemporaryFile = field(repr=False)
 
@@ -86,7 +79,8 @@ class UploadedFile:
     def __del__(self):
         try:
             self._file.close()
-        except Exception:
+        except Exception:  # noqa: BLE001, S110
+            # __del__ must never raise
             pass
 
 
@@ -98,27 +92,27 @@ class FormData:
     """
 
     def __init__(self):
-        self._data: List[Tuple[str, Union[str, UploadedFile]]] = []
+        self._data: list[tuple[str, str | UploadedFile]] = []
 
-    def append(self, key: str, value: Union[str, UploadedFile]) -> None:
+    def append(self, key: str, value: str | UploadedFile) -> None:
         """Add a key-value pair."""
         self._data.append((key, value))
 
-    def __getitem__(self, key: str) -> Union[str, UploadedFile]:
+    def __getitem__(self, key: str) -> str | UploadedFile:
         """Get the first value for a key."""
         for k, v in self._data:
             if k == key:
                 return v
         raise KeyError(key)
 
-    def get(self, key: str, default: Any = None) -> Optional[Union[str, UploadedFile]]:
+    def get(self, key: str, default: Any = None) -> str | UploadedFile | None:
         """Get the first value for a key, or default if not found."""
         try:
             return self[key]
         except KeyError:
             return default
 
-    def getlist(self, key: str) -> List[Union[str, UploadedFile]]:
+    def getlist(self, key: str) -> list[str | UploadedFile]:
         """Get all values for a key."""
         return [v for k, v in self._data if k == key]
 
@@ -142,15 +136,15 @@ class FormData:
         """Return unique keys."""
         return list(self)
 
-    def items(self) -> List[Tuple[str, Union[str, UploadedFile]]]:
+    def items(self) -> list[tuple[str, str | UploadedFile]]:
         """Return all key-value pairs."""
         return list(self._data)
 
-    def values(self) -> List[Union[str, UploadedFile]]:
+    def values(self) -> list[str | UploadedFile]:
         """Return all values."""
         return [v for _, v in self._data]
 
-    def _uploaded_files(self) -> List[UploadedFile]:
+    def _uploaded_files(self) -> list[UploadedFile]:
         """Return UploadedFile instances contained in this form."""
         return [v for _, v in self._data if isinstance(v, UploadedFile)]
 
@@ -163,7 +157,7 @@ class FormData:
         for uploaded in self._uploaded_files():
             try:
                 uploaded.close_sync()
-            except Exception:
+            except Exception:  # noqa: BLE001, S110
                 # Best-effort cleanup; ignore close errors
                 pass
 
@@ -172,7 +166,7 @@ class FormData:
         for uploaded in self._uploaded_files():
             try:
                 await uploaded.close()
-            except Exception:
+            except Exception:  # noqa: BLE001, S110
                 # Best-effort cleanup; ignore close errors
                 pass
 
@@ -189,13 +183,13 @@ class FormData:
         await self.aclose()
 
 
-def parse_content_disposition(header: str) -> Dict[str, Optional[str]]:
+def parse_content_disposition(header: str) -> dict[str, str | None]:
     """
     Parse Content-Disposition header value.
 
     Returns dict with 'name', 'filename' keys (filename may be None).
     """
-    result: Dict[str, Optional[str]] = {"name": None, "filename": None}
+    result: dict[str, str | None] = {"name": None, "filename": None}
 
     # Split on semicolons, handling quoted strings
     parts = []
@@ -238,7 +232,8 @@ def parse_content_disposition(header: str) -> Dict[str, Optional[str]]:
                         from urllib.parse import unquote
 
                         result["filename"] = unquote(encoded, encoding="utf-8")
-                    except Exception:
+                    except Exception:  # noqa: BLE001, S110
+                        # Malformed RFC 5987 filename* - fall back to the plain filename
                         pass
             continue
 
@@ -250,20 +245,19 @@ def parse_content_disposition(header: str) -> Dict[str, Optional[str]]:
 
         if key == "name":
             result["name"] = value
-        elif key == "filename":
+        # Only set filename if filename* hasn't already set it
-            # Only set if filename* hasn't already set it
+        elif key == "filename" and result["filename"] is None:
-            if result["filename"] is None:
+            # Strip path components (security)
-                # Strip path components (security)
+            # Handle both Unix and Windows paths
-                # Handle both Unix and Windows paths
+            value = value.replace("\\", "/")
-                value = value.replace("\\", "/")
+            if "/" in value:
-                if "/" in value:
+                value = value.rsplit("/", 1)[-1]
-                    value = value.rsplit("/", 1)[-1]
+            result["filename"] = value
-                result["filename"] = value
 
     return result
 
 
-def parse_content_type(header: str) -> Tuple[str, Dict[str, str]]:
+def parse_content_type(header: str) -> tuple[str, dict[str, str]]:
     """
     Parse Content-Type header value.
 
@@ -307,7 +301,7 @@ class MultipartParser:
         max_request_size: int = DEFAULT_MAX_REQUEST_SIZE,
         max_fields: int = DEFAULT_MAX_FIELDS,
         max_files: int = DEFAULT_MAX_FILES,
-        max_parts: Optional[int] = DEFAULT_MAX_PARTS,
+        max_parts: int | None = DEFAULT_MAX_PARTS,
         max_field_size: int = DEFAULT_MAX_FIELD_SIZE,
         max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE,
         max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES,
@@ -348,12 +342,12 @@ class MultipartParser:
         self._tempdir = tempfile.gettempdir()
 
         # Current part state
-        self.current_headers: Dict[str, str] = {}
+        self.current_headers: dict[str, str] = {}
-        self.current_file: Optional[tempfile.SpooledTemporaryFile] = None
+        self.current_file: tempfile.SpooledTemporaryFile | None = None
         self.current_body = bytearray()
-        self.current_name: Optional[str] = None
+        self.current_name: str | None = None
-        self.current_filename: Optional[str] = None
+        self.current_filename: str | None = None
-        self.current_content_type: Optional[str] = None
+        self.current_content_type: str | None = None
 
     def feed(self, chunk: bytes) -> None:
         """Feed a chunk of data to the parser."""
@@ -364,6 +358,13 @@ class MultipartParser:
         self.buffer.extend(chunk)
         self._process()
 
+    def close(self) -> None:
+        """Discard completed uploads and any file still being received."""
+        if self.current_file is not None:
+            self.current_file.close()
+            self.current_file = None
+        self.form_data.close()
+
     def _process(self) -> None:
         """Process buffered data."""
         while True:
@@ -454,7 +455,7 @@ class MultipartParser:
             # Parse header
             try:
                 line_str = line.decode("utf-8", errors="replace")
-            except Exception:
+            except UnicodeDecodeError:
                 line_str = line.decode("latin-1")
 
             if ":" in line_str:
@@ -481,7 +482,9 @@ class MultipartParser:
             if self.file_count > self.max_files:
                 raise MultipartParseError("Too many files")
             if self.handle_files:
-                self.current_file = tempfile.SpooledTemporaryFile(
+                # Outlives this method - it is filled in across parser callbacks
+                # and then handed to the UploadedFile the caller consumes
+                self.current_file = tempfile.SpooledTemporaryFile(  # noqa: SIM115
                     max_size=self.max_memory_file_size
                 )
             else:
@@ -581,6 +584,9 @@ class MultipartParser:
     def _finish_part(self) -> None:
         """Finalize current part and add to form data."""
         if self.current_name is None:
+            if self.current_file is not None:
+                self.current_file.close()
+                self.current_file = None
             return
 
         if self.current_filename is not None:
@@ -644,7 +650,7 @@ async def parse_form_data(
     max_request_size: int = DEFAULT_MAX_REQUEST_SIZE,
     max_fields: int = DEFAULT_MAX_FIELDS,
     max_files: int = DEFAULT_MAX_FILES,
-    max_parts: Optional[int] = DEFAULT_MAX_PARTS,
+    max_parts: int | None = DEFAULT_MAX_PARTS,
     max_field_size: int = DEFAULT_MAX_FIELD_SIZE,
     max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE,
     max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES,
@@ -726,29 +732,50 @@ async def parse_form_data(
         batch_target = 64 * 1024
         batch = bytearray()
 
+        async def run_parser(fn, *args):
+            # Cancellation must not close files while a worker is using them.
+            task = asyncio.create_task(asyncio.to_thread(fn, *args))
+            try:
+                return await asyncio.shield(task)
+            except asyncio.CancelledError as cancelled:
+                try:
+                    while not task.done():
+                        try:
+                            await asyncio.shield(task)
+                        except asyncio.CancelledError:
+                            continue
+                    task.result()
+                finally:
+                    raise cancelled
+
         async def flush_batch() -> None:
             if batch:
                 data = bytes(batch)
                 batch.clear()
-                await asyncio.to_thread(parser.feed, data)
+                await run_parser(parser.feed, data)
 
-        while True:
+        try:
-            message = await receive()
+            while True:
-            message_type = message.get("type")
+                message = await receive()
-            if message_type == "http.disconnect":
+                message_type = message.get("type")
-                raise MultipartParseError("Client disconnected during request body")
+                if message_type == "http.disconnect":
-            if message_type is not None and message_type != "http.request":
+                    raise MultipartParseError("Client disconnected during request body")
-                continue
+                if message_type is not None and message_type != "http.request":
-            chunk = message.get("body", b"")
+                    continue
-            if chunk:
+                chunk = message.get("body", b"")
-                batch.extend(chunk)
+                if chunk:
-                if len(batch) >= batch_target:
+                    batch.extend(chunk)
-                    await flush_batch()
+                    if len(batch) >= batch_target:
-            if not message.get("more_body", False):
+                        await flush_batch()
-                break
+                if not message.get("more_body", False):
+                    break
 
-        await flush_batch()
+            await flush_batch()
-        return await asyncio.to_thread(parser.finalize)
+            return await run_parser(parser.finalize)
+        except BaseException:
+            # No FormData is returned to the caller to take ownership on failure.
+            await asyncio.to_thread(parser.close)
+            raise
 
     else:
         raise MultipartParseError(

datasette/utils/permissions.py

@@ -2,8 +2,9 @@
 from __future__ import annotations
 
 import json
-from typing import Any, Dict, Iterable, List, Sequence, Tuple
 import sqlite3
+from collections.abc import Iterable, Sequence
+from typing import Any
 
 from datasette.permissions import PermissionSQL
 from datasette.plugins import pm
@@ -15,7 +16,7 @@ SKIP_PERMISSION_CHECKS = object()
 
 async def gather_permission_sql_from_hooks(
     *, datasette, actor: dict | None, action: str
-) -> List[PermissionSQL] | object:
+) -> list[PermissionSQL] | object:
     """Collect PermissionSQL objects from the permission_resources_sql hook.
 
     Ensures that each returned PermissionSQL has a populated ``source``.
@@ -34,7 +35,7 @@ async def gather_permission_sql_from_hooks(
     hookimpls = hook_caller.get_hookimpls()
     hook_results = list(hook_caller(datasette=datasette, actor=actor, action=action))
 
-    collected: List[PermissionSQL] = []
+    collected: list[PermissionSQL] = []
     actor_json = json.dumps(actor) if actor is not None else None
     actor_id = actor.get("id") if isinstance(actor, dict) else None
 
@@ -71,7 +72,7 @@ def _iter_permission_sql_from_result(
     if isinstance(result, PermissionSQL):
         return [result]
     if isinstance(result, (list, tuple)):
-        collected: List[PermissionSQL] = []
+        collected: list[PermissionSQL] = []
         for item in result:
             collected.extend(_iter_permission_sql_from_result(item, action=action))
         return collected
@@ -90,7 +91,7 @@ def _iter_permission_sql_from_result(
 
 def build_rules_union(
     actor: dict | None, plugins: Sequence[PermissionSQL]
-) -> Tuple[str, Dict[str, Any]]:
+) -> tuple[str, dict[str, Any]]:
     """
     Compose plugin SQL into a UNION ALL.
 
@@ -102,10 +103,10 @@ def build_rules_union(
     The system reserves these parameter names: :actor, :actor_id, :action, :filter_parent
     Plugin parameters should be prefixed with a unique identifier (e.g., source name).
     """
-    parts: List[str] = []
+    parts: list[str] = []
     actor_json = json.dumps(actor) if actor else None
     actor_id = actor.get("id") if actor else None
-    params: Dict[str, Any] = {"actor": actor_json, "actor_id": actor_id}
+    params: dict[str, Any] = {"actor": actor_json, "actor_id": actor_id}
 
     for p in plugins:
         # No namespacing - just use plugin params as-is
@@ -141,10 +142,10 @@ async def resolve_permissions_from_catalog(
     plugins: Sequence[Any],
     action: str,
     candidate_sql: str,
-    candidate_params: Dict[str, Any] | None = None,
+    candidate_params: dict[str, Any] | None = None,
     *,
     implicit_deny: bool = True,
-) -> List[Dict[str, Any]]:
+) -> list[dict[str, Any]]:
     """
     Resolve permissions by embedding the provided *candidate_sql* in a CTE.
 
@@ -168,8 +169,8 @@ async def resolve_permissions_from_catalog(
       - parent, child, allow, reason, source_plugin, depth
       - resource (rendered "/parent/child" or "/parent" or "/")
     """
-    resolved_plugins: List[PermissionSQL] = []
+    resolved_plugins: list[PermissionSQL] = []
-    restriction_sqls: List[str] = []
+    restriction_sqls: list[str] = []
 
     for plugin in plugins:
         if callable(plugin) and not isinstance(plugin, PermissionSQL):
@@ -398,11 +399,11 @@ async def resolve_permissions_with_candidates(
     db,
     actor: dict | None,
     plugins: Sequence[Any],
-    candidates: List[Tuple[str, str | None]],
+    candidates: list[tuple[str, str | None]],
     action: str,
     *,
     implicit_deny: bool = True,
-) -> List[Dict[str, Any]]:
+) -> list[dict[str, Any]]:
     """
     Resolve permissions without any external candidate table by embedding
     the candidates as a UNION of parameterized SELECTs in a CTE.
@@ -411,8 +412,8 @@ async def resolve_permissions_with_candidates(
     actor: actor dict (or None), made available as :actor (JSON), :actor_id, and :action
     """
     # Build a small CTE for candidates.
-    cand_rows_sql: List[str] = []
+    cand_rows_sql: list[str] = []
-    cand_params: Dict[str, Any] = {}
+    cand_params: dict[str, Any] = {}
     for i, (parent, child) in enumerate(candidates):
         pkey = f"cand_p_{i}"
         ckey = f"cand_c_{i}"

datasette/utils/shutil_backport.py

@@ -6,7 +6,7 @@ https://github.com/python/cpython/blob/v3.8.3/LICENSE
 """
 
 import os
-from shutil import copy, copy2, copystat, Error
+from shutil import Error, copy, copy2, copystat
 
 
 def _copytree(

datasette/utils/sql_analysis.py

@@ -1,6 +1,8 @@
+import sys
 from dataclasses import dataclass
 from typing import Literal
 
+from datasette.utils import escape_sqlite
 from datasette.utils.sqlite import SQLiteTableType, sqlite3, sqlite_table_type
 
 SQLOperation = Literal[
@@ -195,6 +197,16 @@ def _allow_authorizer_action(*args):
     return sqlite3.SQLITE_OK
 
 
+def _disable_authorizer(conn):
+    # Python 3.11 added support for unregistering an authorizer using None.
+    # On Python 3.10, None is installed as the callback instead, and the next
+    # statement fails with "not authorized" when sqlite3 tries to call it.
+    if sys.version_info >= (3, 11):
+        conn.set_authorizer(None)
+    else:
+        conn.set_authorizer(_allow_authorizer_action)
+
+
 def analyze_sql_tables(
     conn,
     sql: str,
@@ -208,7 +220,9 @@ def analyze_sql_tables(
 
     This function is synchronous and connection-based. It temporarily installs a
     SQLite authorizer, prepares ``EXPLAIN <sql>``, and returns the operation
-    callbacks observed while SQLite compiles the statement.
+    callbacks observed while SQLite compiles the statement. ``CREATE VIEW`` is
+    additionally executed inside a rolled-back savepoint so its source-table reads
+    can be discovered by analyzing a query against the temporary view.
     """
     operations: dict[OperationKey, set[str]] = {}
 
@@ -413,12 +427,12 @@ def analyze_sql_tables(
                 database=None,
                 table=None,
                 sqlite_schema=sqlite_schema,
-                target="{} {}".format(arg1, arg2) if arg2 is not None else arg1,
+                target=f"{arg1} {arg2}" if arg2 is not None else arg1,
                 source=source,
             )
             return sqlite3.SQLITE_OK
 
-        action_name = _AUTHORIZER_ACTION_NAMES.get(action, "SQLITE_{}".format(action))
+        action_name = _AUTHORIZER_ACTION_NAMES.get(action, f"SQLITE_{action}")
         record(
             "unknown",
             "unknown",
@@ -481,7 +495,7 @@ def analyze_sql_tables(
                         conn, key.table, schema=key.sqlite_schema
                     )
     finally:
-        conn.set_authorizer(None)
+        _disable_authorizer(conn)
 
     has_schema_operation = any(
         key.target_type in {"table", "index", "view", "trigger", "virtual-table"}
@@ -521,9 +535,7 @@ def analyze_sql_tables(
             and key.target in _SQLITE_INTERNAL_SCHEMA_FUNCTIONS
         ):
             return True
-        if key_is_drop_table_delete(key):
+        return bool(key_is_drop_table_delete(key))
-            return True
-        return False
 
     def table_kind_for(key: OperationKey) -> SQLiteTableType | None:
         if (
@@ -534,7 +546,7 @@ def analyze_sql_tables(
             return None
         return table_kind_cache[(key.sqlite_schema, key.table)]
 
-    return SQLAnalysis(
+    analysis = SQLAnalysis(
         operations=tuple(
             Operation(
                 operation=key.operation,
@@ -551,3 +563,58 @@ def analyze_sql_tables(
             for key, columns in operations.items()
         )
     )
+
+    # SQLite does not resolve the SELECT body of a view when preparing CREATE
+    # VIEW, so its authorizer does not report reads from the view's source
+    # tables. Temporarily create the view, analyze a query against it (which
+    # does resolve the body), then roll the schema change back. Database-level
+    # callers use an isolated writable connection for this analysis.
+    create_view_operations = tuple(
+        operation
+        for operation in analysis.operations
+        if operation.operation == "create" and operation.target_type == "view"
+    )
+    if not create_view_operations:
+        return analysis
+
+    savepoint = "datasette_analyze_create_view"
+    conn.execute(f"SAVEPOINT {savepoint}")
+    try:
+        conn.execute(sql, params if params is not None else {})
+        dependency_reads = []
+        for view_operation in create_view_operations:
+            if view_operation.sqlite_schema is None or view_operation.table is None:
+                raise sqlite3.OperationalError(
+                    "Could not determine the created view name"
+                )
+            quoted_schema = escape_sqlite(view_operation.sqlite_schema)
+            quoted_view = escape_sqlite(view_operation.table)
+            qualified_view = f"{quoted_schema}.{quoted_view}"
+            view_analysis = analyze_sql_tables(
+                conn,
+                f"SELECT * FROM {qualified_view}",
+                database_name=database_name,
+                schema_to_database=schema_to_database,
+            )
+            dependency_reads.extend(
+                operation
+                for operation in view_analysis.operations
+                if operation.operation == "read"
+                and not (
+                    operation.sqlite_schema == view_operation.sqlite_schema
+                    and operation.table == view_operation.table
+                )
+            )
+    finally:
+        conn.execute(f"ROLLBACK TO {savepoint}")
+        conn.execute(f"RELEASE {savepoint}")
+
+    existing_operations = set(analysis.operations)
+    return SQLAnalysis(
+        operations=analysis.operations
+        + tuple(
+            operation
+            for operation in dependency_reads
+            if operation not in existing_operations
+        )
+    )

datasette/utils/sqlite.py

@@ -15,8 +15,17 @@ if hasattr(sqlite3, "enable_callback_tracebacks"):
 _cached_sqlite_version = None
 _cached_supports_returning = None
 SQLiteTableType = Literal["table", "view", "virtual", "shadow"]
+_SQLITE_IDENTIFIER_RE = (
+    r"""(?:"(?:[^"]|"")*"|'(?:[^']|'')*'|`(?:[^`]|``)*`|\[[^\]]*\]|[^\s.()'"`\[\]]+)"""
+)
 _VIRTUAL_TABLE_MODULE_RE = re.compile(
-    r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)",
+    r"^\s*CREATE\s+VIRTUAL\s+TABLE\b\s*(?:IF\s+NOT\s+EXISTS\s+)?"
+    + _SQLITE_IDENTIFIER_RE
+    + r"(?:\s*\.\s*"
+    + _SQLITE_IDENTIFIER_RE
+    + r")?\s*\bUSING\b\s*("
+    + _SQLITE_IDENTIFIER_RE
+    + r")",
     re.IGNORECASE | re.DOTALL,
 )
 _VIRTUAL_TABLE_SHADOW_SUFFIXES = {
@@ -83,24 +92,58 @@ def sqlite_table_type(
 ) -> SQLiteTableType | None:
     if supports_table_list():
         try:
-            query = "select type from pragma_table_list where name = ?"
+            # Use the "PRAGMA table_list" statement form rather than the
-            params: tuple[str, ...] = (table,)
+            # pragma_table_list(...) table-valued function. The
+            # table-valued function is resolved like an ordinary relation
+            # name, so an attacker-created table or view literally named
+            # "pragma_table_list" can shadow it and spoof the reported
+            # type (e.g. claiming a virtual table is an ordinary table).
+            # The PRAGMA statement form is a distinct piece of SQL syntax
+            # that always invokes SQLite's built-in pragma, so it cannot
+            # be shadowed by a user-created relation.
             if schema is not None:
-                query += " and schema = ?"
+                query = f"PRAGMA {_quote_identifier(schema)}.table_list"
-                params = (table, schema)
+            else:
-            row = conn.execute(query, params).fetchone()
+                query = "PRAGMA table_list"
-            if row is not None and row[0] in {"table", "view", "virtual", "shadow"}:
+            cursor = conn.execute(query)
-                return row[0]
+            columns = [description[0] for description in cursor.description]
+            for row in cursor.fetchall():
+                record = dict(zip(columns, row))
+                if record.get("name") != table:
+                    continue
+                if schema is not None and record.get("schema") != schema:
+                    continue
+                row_type = record.get("type")
+                if row_type in {"table", "view", "virtual", "shadow"}:
+                    return row_type
         except sqlite3.DatabaseError:
             pass
     return _sqlite_table_type_from_schema(conn, table, schema=schema)
 
 
+def check_structured_write_table(conn, table: str, *, allow_missing=False):
+    """Validate a row-write target on the connection that will perform the write."""
+    # SQLite resolves identifiers case-insensitively. The create API must not
+    # treat a differently cased existing name as a missing table.
+    row = conn.execute(
+        "select name from main.sqlite_master where name = ? collate nocase "
+        "and type in ('table', 'view')",
+        (table,),
+    ).fetchone()
+    if row is None and allow_missing:
+        return
+    if row is not None and sqlite_table_type(conn, row[0]) == "table":
+        return
+    # Virtual table modules can interpret row writes as administrative operations.
+    # Their shadow tables are internal storage, not independently writable data.
+    raise ValueError("Structured writes require an ordinary table")
+
+
 def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]:
     schema_table = _sqlite_schema_table(schema)
     try:
         rows = conn.execute(
-            "select name, sql from {} where type = 'table'".format(schema_table)
+            f"select name, sql from {schema_table} where type = 'table'"
         ).fetchall()
     except sqlite3.DatabaseError:
         return []
@@ -118,6 +161,63 @@ def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]
     return sorted(hidden_tables) + content_fts_tables
 
 
+def sqlite_derived_table_dependencies(
+    conn, *, schema: str | None = "main"
+) -> dict[str, str]:
+    """Return implementation table -> logical/content table dependencies.
+
+    ``PRAGMA table_list`` safely identifies virtual and shadow tables, but
+    does not report which virtual table owns a shadow table or which table is
+    named by an FTS ``content=`` option. Derive those relationships from
+    ``sqlite_master`` DDL and the documented shadow-table suffixes.
+
+    Database errors propagate: failed discovery must not be mistaken for an
+    empty dependency map and cached as permission to skip inheritance.
+    """
+    schema_table = _sqlite_schema_table(schema)
+    rows = conn.execute(
+        f"select name, sql from {schema_table} where type = 'table'"
+    ).fetchall()
+
+    table_names = {row[0] for row in rows}
+    # SQLite identifiers fold ASCII letters only.
+    identifier_case = str.maketrans(
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
+    )
+    canonical_names = {name.translate(identifier_case): name for name in table_names}
+    dependencies = {}
+    for virtual_table, sql in rows:
+        module = _virtual_table_module(sql)
+        if module is None:
+            continue
+
+        # SQLite's documented shadow tables are implementation details of
+        # their logical virtual table.
+        for suffix in _VIRTUAL_TABLE_SHADOW_SUFFIXES.get(module, ()):
+            shadow_table = virtual_table + suffix
+            if shadow_table in table_names:
+                dependencies[shadow_table] = virtual_table
+
+        # An external-content FTS table can expose values fetched from its
+        # content table, so it must also depend on that table's permission.
+        if module in {"fts3", "fts4", "fts5"}:
+            content_table = _fts_external_content_table(sql)
+            if content_table:
+                dependencies[virtual_table] = content_table
+
+        if module in {"fts5vocab", "fts4aux"}:
+            source = _fts_vocabulary_source(sql, module, schema or "main")
+            source = (
+                canonical_names.get(source.translate(identifier_case))
+                if source
+                else None
+            )
+            # An unresolved source is itself derived, so the one-hop policy denies it.
+            dependencies[virtual_table] = source or virtual_table
+
+    return dependencies
+
+
 def _sqlite_table_type_from_schema(
     conn,
     table: str,
@@ -127,7 +227,7 @@ def _sqlite_table_type_from_schema(
     schema_table = _sqlite_schema_table(schema)
     try:
         row = conn.execute(
-            "select type, sql from {} where name = ?".format(schema_table),
+            f"select type, sql from {schema_table} where name = ?",
             (table,),
         ).fetchone()
     except sqlite3.DatabaseError:
@@ -155,7 +255,7 @@ def _is_known_shadow_table(
     schema_table = _sqlite_schema_table(schema)
     try:
         rows = conn.execute(
-            "select name, sql from {} where type = 'table'".format(schema_table)
+            f"select name, sql from {schema_table} where type = 'table'"
         ).fetchall()
     except sqlite3.DatabaseError:
         return False
@@ -174,7 +274,7 @@ def _sqlite_schema_table(schema: str | None) -> str:
         return "sqlite_master"
     if schema == "temp":
         return "sqlite_temp_master"
-    return "{}.sqlite_master".format(_quote_identifier(schema))
+    return f"{_quote_identifier(schema)}.sqlite_master"
 
 
 def _quote_identifier(value: str) -> str:
@@ -184,10 +284,151 @@ def _quote_identifier(value: str) -> str:
 def _virtual_table_module(sql: str | None) -> str | None:
     if not sql:
         return None
+    match = _VIRTUAL_TABLE_MODULE_RE.search(_strip_sql_comments(sql))
+    if match is None:
+        return None
+    return _unquote_sql_value(match.group(1)).lower()
+
+
+def _fts_external_content_table(sql: str | None) -> str | None:
+    """Extract the external ``content=`` table from an FTS declaration."""
+    if not sql:
+        return None
+    sql = _strip_sql_comments(sql)
     match = _VIRTUAL_TABLE_MODULE_RE.search(sql)
     if match is None:
         return None
-    return match.group(1).strip("\"'[]`").lower()
+    open_paren = sql.find("(", match.end())
+    if open_paren == -1:
+        return None
+    close_paren = sql.rfind(")")
+    if close_paren <= open_paren:
+        return None
+
+    for argument in _split_sql_arguments(sql[open_paren + 1 : close_paren]):
+        key, separator, value = argument.partition("=")
+        if not separator or key.strip().lower() != "content":
+            continue
+        return _unquote_sql_value(value.strip())
+    return None
+
+
+def _fts_vocabulary_source(sql: str, module: str, schema: str) -> str | None:
+    """Resolve a vocabulary source within the current SQLite schema.
+
+    Cross-schema sources cannot be represented by the dependency map and
+    are conservatively left unresolved.
+    """
+    sql = _strip_sql_comments(sql)
+    match = _VIRTUAL_TABLE_MODULE_RE.search(sql)
+    if match is None:
+        return None
+    start = sql.find("(", match.end())
+    end = sql.rfind(")")
+    if start < 0 or end <= start:
+        return None
+    arguments = [
+        _unquote_sql_value(arg.strip())
+        for arg in _split_sql_arguments(sql[start + 1 : end])
+    ]
+    expected = 2 if module == "fts5vocab" else 1
+    if len(arguments) == expected:
+        return arguments[0]
+    if len(arguments) == expected + 1 and arguments[0].lower() == schema.lower():
+        return arguments[1]
+    return None
+
+
+def _split_sql_arguments(arguments: str) -> list[str]:
+    """Split comma-separated SQLite arguments without splitting quoted text."""
+    parts = []
+    start = 0
+    quote = None
+    closing_quote = None
+    index = 0
+    while index < len(arguments):
+        char = arguments[index]
+        if quote is None:
+            if char in {"'", '"', "`", "["}:
+                quote = char
+                closing_quote = "]" if char == "[" else char
+            elif char == ",":
+                parts.append(arguments[start:index])
+                start = index + 1
+        elif char == closing_quote:
+            # Single/double/backtick quoting escapes the delimiter by
+            # doubling it. Square-bracket identifiers do not.
+            if (
+                quote != "["
+                and index + 1 < len(arguments)
+                and arguments[index + 1] == closing_quote
+            ):
+                index += 1
+            else:
+                quote = None
+                closing_quote = None
+        index += 1
+    parts.append(arguments[start:])
+    return parts
+
+
+def _strip_sql_comments(sql: str) -> str:
+    """Remove SQLite comments while preserving quoted strings/identifiers."""
+    output = []
+    quote = None
+    closing_quote = None
+    index = 0
+    while index < len(sql):
+        char = sql[index]
+        next_char = sql[index + 1] if index + 1 < len(sql) else ""
+        if quote is None:
+            if char in {"'", '"', "`", "["}:
+                quote = char
+                closing_quote = "]" if char == "[" else char
+                output.append(char)
+            elif char == "-" and next_char == "-":
+                index += 2
+                while index < len(sql) and sql[index] not in "\r\n":
+                    index += 1
+                output.append(" ")
+                continue
+            elif char == "/" and next_char == "*":
+                index += 2
+                while index + 1 < len(sql) and sql[index : index + 2] != "*/":
+                    index += 1
+                index = min(index + 2, len(sql))
+                output.append(" ")
+                continue
+            else:
+                output.append(char)
+        else:
+            output.append(char)
+            if char == closing_quote:
+                if (
+                    quote != "["
+                    and index + 1 < len(sql)
+                    and sql[index + 1] == closing_quote
+                ):
+                    output.append(sql[index + 1])
+                    index += 1
+                else:
+                    quote = None
+                    closing_quote = None
+        index += 1
+    return "".join(output)
+
+
+def _unquote_sql_value(value: str) -> str:
+    if len(value) < 2:
+        return value
+    pairs = {"'": "'", '"': '"', "`": "`", "[": "]"}
+    closing = pairs.get(value[0])
+    if closing is None or value[-1] != closing:
+        return value
+    unquoted = value[1:-1]
+    if value[0] != "[":
+        unquoted = unquoted.replace(closing * 2, closing)
+    return unquoted
 
 
 def _is_fts_content_virtual_table(sql: str | None) -> bool:

datasette/utils/testing.py

@@ -1,9 +1,10 @@
-from asgiref.sync import async_to_sync
-from urllib.parse import urlencode
 import json
+from urllib.parse import urlencode
+
+from asgiref.sync import async_to_sync
 
 # These wrapper classes pre-date the introduction of
-# datasette.client and httpx to Datasette. They could
+# datasette.client and httpx2 to Datasette. They could
 # be removed if the Datasette tests are modified to
 # call datasette.client directly.
 

datasette/version.py

@@ -1,2 +1,2 @@
-__version__ = "1.0a36"
+__version__ = "1.0a41"
 __version_info__ = tuple(__version__.split("."))

datasette/views/__init__.py

@@ -1,7 +1,7 @@
-from dataclasses import dataclass
 import dataclasses
 import types
 import typing
+from dataclasses import dataclass
 
 
 @dataclass(frozen=True)
@@ -74,16 +74,14 @@ class Context:
             extra_class = table_extra_registry.classes_by_name[name]
         except KeyError:
             raise KeyError(
-                "{}.{} is declared with from_extra() but there is no "
+                f"{cls.__name__}.{name} is declared with from_extra() but there is no "
-                "registered extra of that name".format(cls.__name__, name)
+                "registered extra of that name"
             )
         if cls.extras_scope is not None and not extra_class.available_for(
             cls.extras_scope
         ):
             raise ValueError(
-                "{}.{} is declared with from_extra() but the {} extra is "
+                f"{cls.__name__}.{name} is declared with from_extra() but the {name} extra is "
-                "not available for scope {}".format(
+                f"not available for scope {cls.extras_scope}"
-                    cls.__name__, name, name, cls.extras_scope
-                )
             )
         return extra_class.description or ""

datasette/views/base.py

@@ -2,20 +2,20 @@ import csv
 import hashlib
 import sys
 
-from datasette.utils.asgi import Request
 from datasette.utils import (
-    add_cors_headers,
     EscapeHtmlWriter,
     InvalidSql,
     LimitedWriter,
+    add_cors_headers,
     path_from_row_pks,
     path_with_format,
     sqlite3,
 )
 from datasette.utils.asgi import (
     AsgiStream,
-    Response,
     BadRequest,
+    Request,
+    Response,
 )
 
 
@@ -35,7 +35,7 @@ class DatasetteError(Exception):
         self.error_dict = error_dict or {}
         self.status = status
         self.message_is_html = message_is_html
-        # Plain text used for JSON error responses when message is HTML
+        # Plain text used for JSON and CSV error responses when message is HTML
         self.plain_message = plain_message
 
 
@@ -129,12 +129,10 @@ class BaseView:
         template = environment.select_template(templates)
         template_context = {
             **context,
-            **{
+            "select_templates": [
-                "select_templates": [
+                f"{'*' if template_name == template.name else ''}{template_name}"
-                    f"{'*' if template_name == template.name else ''}{template_name}"
+                for template_name in templates
-                    for template_name in templates
+            ],
-                ],
-            },
         }
         headers = {}
         if self.has_json_alternate:
@@ -151,9 +149,7 @@ class BaseView:
             template_context["alternate_url_json"] = alternate_url_json
             headers.update(
                 {
-                    "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format(
+                    "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"'
-                        alternate_url_json
-                    )
                 }
             )
         return Response.html(
@@ -184,9 +180,7 @@ async def stream_csv(datasette, fetch_data, request, database):
     stream = request.args.get("_stream")
     # Do not calculate facets or counts:
     extra_parameters = [
-        "{}=1".format(key)
+        f"{key}=1" for key in ("_nofacet", "_nocount") if not request.args.get(key)
-        for key in ("_nofacet", "_nocount")
-        if not request.args.get(key)
     ]
     if extra_parameters:
         # Replace request object with a new one with modified scope
@@ -216,9 +210,6 @@ async def stream_csv(datasette, fetch_data, request, database):
     except (sqlite3.OperationalError, InvalidSql) as e:
         raise DatasetteError(str(e), title="Invalid SQL", status=400)
 
-    except sqlite3.OperationalError as e:
-        raise DatasetteError(str(e))
-
     except DatasetteError:
         raise
 
@@ -325,8 +316,9 @@ async def stream_csv(datasette, fetch_data, request, database):
                             else:
                                 new_row.append(cell)
                         await writer.writerow(new_row)
-            except Exception as ex:
+            except Exception as ex:  # noqa: BLE001
-                sys.stderr.write("Caught this error: {}\n".format(ex))
+                # Streaming CSV: report the error into the response body and stop
+                sys.stderr.write(f"Caught this error: {ex}\n")
                 sys.stderr.flush()
                 await r.write(str(ex))
                 return

datasette/views/database.py

@@ -1,49 +1,56 @@
-from dataclasses import asdict, dataclass, field
-from urllib.parse import parse_qsl, urlencode
 import asyncio
 import hashlib
 import itertools
 import json
-import markupsafe
 import os
 import textwrap
+from dataclasses import asdict, dataclass, field
+from urllib.parse import parse_qsl, urlencode
+
+import markupsafe
 
-from datasette.extras import extra_names_from_request, ExtraScope
 from datasette.database import QueryInterrupted
+from datasette.extras import ExtraScope, extra_names_from_request
+from datasette.plugins import pm
 from datasette.resources import DatabaseResource, QueryResource
 from datasette.stored_queries import StoredQuery, stored_query_to_dict
-from datasette.write_sql import QueryWriteRejected
 from datasette.utils import (
+    InvalidSql,
     add_cors_headers,
     await_me_maybe,
-    error_body,
     call_with_supported_arguments,
-    named_parameters as derive_named_parameters,
+    error_body,
     format_bytes,
-    make_slot_function,
-    tilde_decode,
-    to_css_class,
-    validate_sql_select,
     is_url,
+    make_slot_function,
     path_with_added_args,
     path_with_format,
     path_with_removed_args,
     sqlite3,
+    tilde_decode,
+    to_css_class,
     truncate_url,
-    InvalidSql,
+    validate_sql_select,
 )
-from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden
+from datasette.utils import (
-from datasette.plugins import pm
+    named_parameters as derive_named_parameters,
+)
+from datasette.utils.asgi import AsgiFileDownload, Forbidden, NotFound, Response
+from datasette.write_sql import QueryWriteRejected
 
+from . import Context
 from .base import DatasetteError, View, stream_csv
-from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns
+from .query_helpers import (
+    _block_framing,
+    _ensure_stored_query_execution_permissions,
+    _table_columns,
+)
+from .table_create_alter import _create_table_ui_context
 from .table_extras import (
     QueryExtraContext,
     resolve_query_extras,
     table_extra_registry,
 )
-from .table_create_alter import _create_table_ui_context
-from . import Context
 
 
 @dataclass
@@ -100,7 +107,7 @@ class DatabaseView(View):
             return response
 
         if format_ not in ("html", "json"):
-            raise NotFound("Invalid format: {}".format(format_))
+            raise NotFound(f"Invalid format: {format_}")
 
         metadata = await datasette.get_database_metadata(database)
 
@@ -164,7 +171,7 @@ class DatabaseView(View):
                         "label": "Create table",
                         "description": "Create a new table in this database.",
                         "attrs": {
-                            "aria-label": "Create table in {}".format(database),
+                            "aria-label": f"Create table in {database}",
                             "data-database-action": "create-table",
                         },
                     }
@@ -271,9 +278,7 @@ class DatabaseView(View):
                 view_name="database",
             ),
             headers={
-                "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format(
+                "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"'
-                    alternate_url_json
-                )
             },
         )
 
@@ -556,7 +561,7 @@ async def database_download(request, datasette):
     if datasette.cors:
         add_cors_headers(headers)
     if db.hash:
-        etag = '"{}"'.format(db.hash)
+        etag = f'"{db.hash}"'
         headers["Etag"] = etag
         # Has user seen this already?
         if_none_match = request.headers.get("if-none-match")
@@ -643,8 +648,15 @@ class QueryView(View):
         ok = None
         redirect_url = None
         try:
+            execute_write_kwargs = {"request": request}
+            if stored_query.is_trusted:
+                analysis = await db.analyze_sql(stored_query.sql, params_for_query)
+                if any(
+                    operation.operation == "vacuum" for operation in analysis.operations
+                ):
+                    execute_write_kwargs["transaction"] = False
             cursor = await db.execute_write(
-                stored_query.sql, params_for_query, request=request
+                stored_query.sql, params_for_query, **execute_write_kwargs
             )
             # success message can come from on_success_message or on_success_message_sql
             message = None
@@ -657,8 +669,9 @@ class QueryView(View):
                     ).first()
                     if message_result:
                         message = message_result[0]
-                except Exception as ex:
+                except Exception as ex:  # noqa: BLE001
-                    message = "Error running on_success_message_sql: {}".format(ex)
+                    # Stored-query on_success_message_sql is user-authored
+                    message = f"Error running on_success_message_sql: {ex}"
                     message_type = datasette.ERROR
             if not message:
                 if stored_query.on_success_message:
@@ -672,7 +685,8 @@ class QueryView(View):
 
             redirect_url = stored_query.on_success_redirect
             ok = True
-        except Exception as ex:
+        except Exception as ex:  # noqa: BLE001
+            # Stored-query execution is user-authored SQL
             message = stored_query.on_error_message or str(ex)
             message_type = datasette.ERROR
             redirect_url = stored_query.on_error_redirect
@@ -806,16 +820,16 @@ class QueryView(View):
                 rows = results.rows
             except QueryInterrupted as ex:
                 raise DatasetteError(
-                    textwrap.dedent("""
+                    textwrap.dedent(f"""
                     <p>SQL query took too long. The time limit is controlled by the
                     <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>
                     configuration option.</p>
-                    <textarea style="width: 90%">{}</textarea>
+                    <textarea style="width: 90%">{markupsafe.escape(ex.sql)}</textarea>
                     <script>
                     let ta = document.querySelector("textarea");
                     ta.style.height = ta.scrollHeight + "px";
                     </script>
-                """.format(markupsafe.escape(ex.sql))).strip(),
+                """).strip(),
                     title="SQL Interrupted",
                     status=400,
                     message_is_html=True,
@@ -831,8 +845,6 @@ class QueryView(View):
                 columns = []
             except (sqlite3.OperationalError, InvalidSql) as ex:
                 raise DatasetteError(str(ex), title="Invalid SQL", status=400)
-            except sqlite3.OperationalError as ex:
-                raise DatasetteError(str(ex))
             except DatasetteError:
                 raise
 
@@ -849,12 +861,13 @@ class QueryView(View):
                 raise DatasetteError("?sql= is required", status=400)
 
             async def fetch_data_for_csv(request, _next=None):
-                results = await db.execute(sql, params, truncate=True)
+                # Reuse the trusted magic parameter values prepared above.
+                results = await db.execute(sql, params_for_query, truncate=True)
                 data = {"rows": results.rows, "columns": results.columns}
                 return data, None, None
 
             return await stream_csv(datasette, fetch_data_for_csv, request, db.name)
-        elif format_ in datasette.renderers.keys():
+        elif format_ in datasette.renderers:
             if not sql:
                 raise DatasetteError("?sql= is required", status=400)
             data = {"ok": True, "rows": rows, "columns": columns}
@@ -946,9 +959,7 @@ class QueryView(View):
             }
             headers.update(
                 {
-                    "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format(
+                    "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"'
-                        alternate_url_json
-                    )
                 }
             )
             metadata = await query_metadata()
@@ -1029,9 +1040,7 @@ class QueryView(View):
                         + "?"
                         + urlencode(
                             {
-                                **{
+                                "sql": sql,
-                                    "sql": sql,
-                                },
                                 **named_parameter_values,
                             }
                         )
@@ -1133,9 +1142,11 @@ class QueryView(View):
                 headers=headers,
             )
         else:
-            assert False, "Invalid format: {}".format(format_)
+            assert False, f"Invalid format: {format_}"
         if datasette.cors:
             add_cors_headers(r.headers)
+        if stored_query_write and format_ == "html":
+            _block_framing(r)
         return r
 
 
@@ -1234,7 +1245,7 @@ async def display_rows(datasette, database, request, rows, columns):
                         '<a class="blob-download" href="{}"{}>&lt;Binary:&nbsp;{:,}&nbsp;byte{}&gt;</a>'.format(
                             blob_url,
                             (
-                                ' title="{}"'.format(formatted)
+                                f' title="{formatted}"'
                                 if "bytes" not in formatted
                                 else ""
                             ),

datasette/views/execute_write.py

@@ -1,6 +1,7 @@
 import re
 from urllib.parse import urlencode
 
+from datasette.database import QueryInterrupted
 from datasette.resources import DatabaseResource
 from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3
 from datasette.utils.asgi import Response
@@ -8,8 +9,8 @@ from datasette.utils.asgi import Response
 from .base import BaseView
 from .database import display_rows as display_query_rows
 from .query_helpers import (
-    QueryValidationError,
     SQL_PARAMETER_FORM_PREFIX,
+    QueryValidationError,
     _analysis_is_write,
     _analysis_rows,
     _analysis_rows_with_permissions,
@@ -31,15 +32,7 @@ WRITE_TEMPLATE_LABELS = {
     "delete": "Delete rows",
 }
 WRITE_TEMPLATE_OPERATIONS = tuple(WRITE_TEMPLATE_LABELS)
-CREATE_TABLE_TEMPLATE_SQL = "\n".join(
+CREATE_TABLE_TEMPLATE_SQL = "create table new_table (\n  id integer primary key,\n  name text\n  -- created text default (datetime('now'))\n)"
-    (
-        "create table new_table (",
-        "  id integer primary key,",
-        "  name text",
-        "  -- created text default (datetime('now'))",
-        ")",
-    )
-)
 
 
 def _parameter_names(columns):
@@ -49,11 +42,11 @@ def _parameter_names(columns):
         base = re.sub(r"[^a-z0-9_]+", "_", column.lower())
         base = base.strip("_") or "value"
         if base[0].isdigit():
-            base = "p_{}".format(base)
+            base = f"p_{base}"
         name = base
         index = 2
         while name in seen:
-            name = "{}_{}".format(base, index)
+            name = f"{base}_{index}"
             index += 1
         seen.add(name)
         names[column] = name
@@ -65,7 +58,7 @@ def _quote_identifier(identifier):
 
 
 def _preferred_where_column(table, columns):
-    lower_table_id = "{}_id".format(table.lower())
+    lower_table_id = f"{table.lower()}_id"
     return (
         next((column for column in columns if column.lower() == "id"), None)
         or next(
@@ -90,17 +83,15 @@ def _insert_template_sql(table, columns):
     auto_pk = _auto_incrementing_primary_key(columns)
     insert_columns = [column for column in column_names if column != auto_pk]
     if not insert_columns:
-        return "insert into {}\ndefault values".format(_quote_identifier(table))
+        return f"insert into {_quote_identifier(table)}\ndefault values"
     names = _parameter_names(insert_columns)
     return "\n".join(
         (
-            "insert into {} (".format(_quote_identifier(table)),
+            f"insert into {_quote_identifier(table)} (",
-            ",\n".join(
+            ",\n".join(f"  {_quote_identifier(column)}" for column in insert_columns),
-                "  {}".format(_quote_identifier(column)) for column in insert_columns
-            ),
             ")",
             "values (",
-            ",\n".join("  :{}".format(names[column]) for column in insert_columns),
+            ",\n".join(f"  :{names[column]}" for column in insert_columns),
             ")",
         )
     )
@@ -114,18 +105,14 @@ def _update_template_sql(table, columns):
     if not set_columns:
         return "\n".join(
             (
-                "update {}".format(_quote_identifier(table)),
+                f"update {_quote_identifier(table)}",
-                "set {} = :new_{}".format(
+                f"set {_quote_identifier(where_column)} = :new_{names[where_column]}",
-                    _quote_identifier(where_column), names[where_column]
+                f"where {_quote_identifier(where_column)} = :{names[where_column]}",
-                ),
-                "where {} = :{}".format(
-                    _quote_identifier(where_column), names[where_column]
-                ),
             )
         )
     return "\n".join(
         (
-            "update {}".format(_quote_identifier(table)),
+            f"update {_quote_identifier(table)}",
             "set "
             + ",\n".join(
                 "{}{} = :{}".format(
@@ -135,9 +122,7 @@ def _update_template_sql(table, columns):
                 )
                 for index, column in enumerate(set_columns)
             ),
-            "where {} = :{}".format(
+            f"where {_quote_identifier(where_column)} = :{names[where_column]}",
-                _quote_identifier(where_column), names[where_column]
-            ),
         )
     )
 
@@ -148,10 +133,8 @@ def _delete_template_sql(table, columns):
     where_column = _preferred_where_column(table, column_names)
     return "\n".join(
         (
-            "delete from {}".format(_quote_identifier(table)),
+            f"delete from {_quote_identifier(table)}",
-            "where {} = :{}".format(
+            f"where {_quote_identifier(where_column)} = :{names[where_column]}",
-                _quote_identifier(where_column), names[where_column]
-            ),
         )
     )
 
@@ -402,7 +385,7 @@ class ExecuteWriteView(BaseView):
         try:
             execute_write_kwargs = {"request": request}
             cursor = await db.execute_write(sql, params, **execute_write_kwargs)
-        except sqlite3.DatabaseError as ex:
+        except (QueryInterrupted, sqlite3.DatabaseError) as ex:
             message = str(ex)
             if wants_json:
                 return _block_framing(Response.error([message], 400))

datasette/views/index.py

@@ -2,11 +2,11 @@ import json
 
 from datasette.plugins import pm
 from datasette.utils import (
+    UNSTABLE_API_MESSAGE,
+    CustomJSONEncoder,
     add_cors_headers,
     await_me_maybe,
     make_slot_function,
-    CustomJSONEncoder,
-    UNSTABLE_API_MESSAGE,
 )
 from datasette.utils.asgi import Response
 from datasette.version import __version__
@@ -46,15 +46,15 @@ class IndexView(BaseView):
 
         databases = []
         # Iterate over allowed databases instead of all databases
-        for name in allowed_db_dict.keys():
+        for name, allowed_db in allowed_db_dict.items():
             db = self.ds.databases[name]
-            database_private = allowed_db_dict[name].private
+            database_private = allowed_db.private
 
             # Get allowed tables/views for this database
             allowed_for_db = tables_by_db.get(name, {})
 
             # Get table names from allowed set instead of db.table_names()
-            table_names = [child_name for child_name in allowed_for_db.keys()]
+            table_names = [child_name for child_name in allowed_for_db]
 
             hidden_table_names = set(await db.hidden_table_names())
 
@@ -99,7 +99,7 @@ class IndexView(BaseView):
                 # We will be sorting by number of relationships, so populate that field
                 all_foreign_keys = await db.get_all_foreign_keys()
                 for table, foreign_keys in all_foreign_keys.items():
-                    if table in tables.keys():
+                    if table in tables:
                         count = len(foreign_keys["incoming"] + foreign_keys["outgoing"])
                         tables[table]["num_relationships_for_sorting"] = count
 
@@ -121,8 +121,7 @@ class IndexView(BaseView):
             # Only add views if this is less than TRUNCATE_AT
             if len(tables_and_views_truncated) < TRUNCATE_AT:
                 num_views_to_add = TRUNCATE_AT - len(tables_and_views_truncated)
-                for view in views[:num_views_to_add]:
+                tables_and_views_truncated.extend(views[:num_views_to_add])
-                    tables_and_views_truncated.append(view)
 
             databases.append(
                 {

datasette/views/query_helpers.py

@@ -5,6 +5,19 @@ from datasette.resources import DatabaseResource
 from datasette.stored_queries import (
     StoredQuery,
 )
+from datasette.utils import (
+    InvalidSql,
+    escape_sqlite,
+    parse_size_limit,
+    path_from_row_pks,
+    sqlite3,
+    validate_sql_select,
+)
+from datasette.utils import (
+    named_parameters as derive_named_parameters,
+)
+from datasette.utils.asgi import Forbidden
+from datasette.utils.sql_analysis import Operation, SQLAnalysis
 from datasette.write_sql import (
     IgnoreWriteSqlOperation,
     QueryWriteRejected,
@@ -12,17 +25,6 @@ from datasette.write_sql import (
     decision_for_write_sql_operation,
     operation_is_write,
 )
-from datasette.utils import (
-    parse_size_limit,
-    named_parameters as derive_named_parameters,
-    escape_sqlite,
-    path_from_row_pks,
-    sqlite3,
-    validate_sql_select,
-    InvalidSql,
-)
-from datasette.utils.asgi import Forbidden
-from datasette.utils.sql_analysis import Operation, SQLAnalysis
 
 _query_name_re = re.compile(r"^[^/\.\n]+$")
 
@@ -91,7 +93,7 @@ def _as_optional_bool(value, name):
             return True
         if lowered in {"0", "false", "f", "no", "off"}:
             return False
-    raise QueryValidationError("{} must be 0 or 1".format(name))
+    raise QueryValidationError(f"{name} must be 0 or 1")
 
 
 def _query_list_limit(value, default, maximum):
@@ -171,7 +173,7 @@ async def _json_or_form_payload(request):
         try:
             return json.loads(body or b"{}"), True
         except json.JSONDecodeError as e:
-            raise QueryValidationError("Invalid JSON: {}".format(e))
+            raise QueryValidationError(f"Invalid JSON: {e}")
     return await request.post_vars(), False
 
 
@@ -192,7 +194,7 @@ async def _analyze_user_query(datasette, db, sql, *, actor):
     try:
         analysis = await db.analyze_sql(sql, params)
     except sqlite3.DatabaseError as ex:
-        raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex
+        raise QueryValidationError(f"Could not analyze query: {ex}") from ex
 
     is_write = _analysis_is_write(analysis)
     if is_write:
@@ -293,8 +295,7 @@ def _coerce_execute_write_payload(data, is_json):
         for key, value in data.items():
             if key in {"sql", "csrftoken", "_json"}:
                 continue
-            if key.startswith(SQL_PARAMETER_FORM_PREFIX):
+            key = key.removeprefix(SQL_PARAMETER_FORM_PREFIX)
-                key = key[len(SQL_PARAMETER_FORM_PREFIX) :]
             params[key] = value
     if not isinstance(params, dict):
         raise QueryValidationError("params must be a dictionary")
@@ -314,7 +315,7 @@ async def _prepare_execute_write(datasette, db, sql, params, actor):
     try:
         analysis = await db.analyze_sql(sql, params)
     except sqlite3.DatabaseError as ex:
-        raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex
+        raise QueryValidationError(f"Could not analyze query: {ex}") from ex
     if not _analysis_is_write(analysis):
         raise QueryValidationError(
             "Use /-/query for read-only SQL; this endpoint only executes writes"
@@ -496,7 +497,7 @@ async def _inserted_row_url(datasette, db, analysis, cursor):
     )
     try:
         result = await db.execute(
-            "select {} from {} where rowid = ?".format(select, escape_sqlite(table)),
+            f"select {select} from {escape_sqlite(table)} where rowid = ?",
             [lastrowid],
         )
     except sqlite3.DatabaseError:

datasette/views/row.py

@@ -8,34 +8,37 @@ from dataclasses import dataclass, field
 import markupsafe
 import sqlite_utils
 
-from datasette.utils.asgi import NotFound, Forbidden, PayloadTooLarge, Response
 from datasette.database import QueryInterrupted
-from datasette.events import UpdateRowEvent, DeleteRowEvent
+from datasette.events import DeleteRowEvent, UpdateRowEvent
+from datasette.extras import ExtraScope, extra_names_from_request
+from datasette.plugins import pm
 from datasette.resources import TableResource
-from .base import BaseView, DatasetteError, stream_csv
 from datasette.utils import (
+    CustomJSONEncoder,
+    CustomRow,
+    InvalidSql,
+    WriteJsonValueError,
     add_cors_headers,
     await_me_maybe,
     call_with_supported_arguments,
-    CustomJSONEncoder,
-    CustomRow,
     decode_write_json_row,
-    InvalidSql,
+    escape_sqlite,
     make_slot_function,
     path_from_row_pks,
     path_with_format,
     path_with_removed_args,
-    to_css_class,
-    escape_sqlite,
     sqlite3,
-    WriteJsonValueError,
+    tilde_decode,
+    to_css_class,
 )
-from datasette.plugins import pm
+from datasette.utils.asgi import Forbidden, NotFound, PayloadTooLarge, Response
-from datasette.extras import extra_names_from_request, ExtraScope
+from datasette.utils.sqlite import check_structured_write_table
+
 from . import Context, from_extra
+from .base import BaseView, DatasetteError, stream_csv
 from .table import (
-    display_columns_and_rows,
     _table_page_data,
+    display_columns_and_rows,
     row_label_from_label_column,
 )
 from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry
@@ -136,6 +139,12 @@ class RowContext(Context):
     )
 
 
+async def _database_and_table_resource_from_request(datasette, request):
+    db = await datasette.resolve_database(request)
+    table = tilde_decode(request.url_vars["table"])
+    return db, table, TableResource(database=db.name, table=table)
+
+
 class RowView(BaseView):
     name = "row"
 
@@ -187,16 +196,16 @@ class RowView(BaseView):
                 data, extra_template_data, templates = response_or_template_contexts
         except QueryInterrupted as ex:
             raise DatasetteError(
-                textwrap.dedent("""
+                textwrap.dedent(f"""
                 <p>SQL query took too long. The time limit is controlled by the
                 <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>
                 configuration option.</p>
-                <textarea style="width: 90%">{}</textarea>
+                <textarea style="width: 90%">{markupsafe.escape(ex.sql)}</textarea>
                 <script>
                 let ta = document.querySelector("textarea");
                 ta.style.height = ta.scrollHeight + "px";
                 </script>
-            """.format(markupsafe.escape(ex.sql))).strip(),
+            """).strip(),
                 title="SQL Interrupted",
                 status=400,
                 message_is_html=True,
@@ -207,15 +216,13 @@ class RowView(BaseView):
             )
         except (sqlite3.OperationalError, InvalidSql) as e:
             raise DatasetteError(str(e), title="Invalid SQL", status=400)
-        except sqlite3.OperationalError as e:
-            raise DatasetteError(str(e))
         except DatasetteError:
             raise
 
         end = time.perf_counter()
         data["query_ms"] = (end - start) * 1000
 
-        if format_ in self.ds.renderers.keys():
+        if format_ in self.ds.renderers:
             # Dispatch request to the correct output format renderer
             # (CSV is not handled here due to streaming)
             result = call_with_supported_arguments(
@@ -258,13 +265,13 @@ class RowView(BaseView):
             if status_code is not None:
                 response.status = status_code
         else:
-            raise NotFound("Invalid format: {}".format(format_))
+            raise NotFound(f"Invalid format: {format_}")
 
         ttl = request.args.get("_ttl", None)
         if ttl is None or not ttl.isdigit():
             ttl = self.ds.setting("default_cache_ttl")
 
-        return self.set_response_headers(response, ttl)
+        return self.set_response_headers(response, ttl, request)
 
     async def html(self, request, data, extra_template_data, templates):
         extras = {}
@@ -373,42 +380,54 @@ class RowView(BaseView):
                 view_name=self.name,
             ),
             headers={
-                "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format(
+                "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"'
-                    alternate_url_json
-                )
             },
         )
 
-    def set_response_headers(self, response, ttl):
+    def set_response_headers(self, response, ttl, request=None):
+        private = getattr(request, "_datasette_private_response", False)
         # Set far-future cache expiry
         if self.ds.cache_headers and response.status == 200:
-            ttl = int(ttl)
+            if private:
-            if ttl == 0:
+                # This response is only visible to the current actor (denied
-                ttl_header = "no-cache"
+                # to anonymous requests), so it must never be stored by a
+                # shared cache/CDN - and ?_ttl= must not override that.
+                response.headers["Cache-Control"] = "private, no-store"
+                response.headers["Vary"] = "Cookie"
             else:
-                ttl_header = f"max-age={ttl}"
+                ttl = int(ttl)
-            response.headers["Cache-Control"] = ttl_header
+                if ttl == 0:
+                    ttl_header = "no-cache"
+                else:
+                    ttl_header = f"max-age={ttl}"
+                response.headers["Cache-Control"] = ttl_header
         response.headers["Referrer-Policy"] = "no-referrer"
         if self.ds.cors:
             add_cors_headers(response.headers)
         return response
 
     async def data(self, request, default_labels=False):
-        resolved = await self.ds.resolve_row(request)
+        db, table, resource = await _database_and_table_resource_from_request(
-        db = resolved.db
+            self.ds, request
+        )
         database = db.name
-        table = resolved.table
-        pk_values = resolved.pk_values
 
-        # Ensure user has permission to view this row
+        # Check the URL resource before resolving the row, so a denied request
+        # cannot distinguish an existing primary key from a missing one.
         visible, private = await self.ds.check_visibility(
             request.actor,
             action="view-table",
-            resource=TableResource(database=database, table=table),
+            resource=resource,
         )
         if not visible:
             raise Forbidden("You do not have permission to view this table")
+        # Record whether this response is private (visible to this actor
+        # only) so set_response_headers() can set appropriate Cache-Control
+        # headers, regardless of which output format ends up being rendered.
+        request._datasette_private_response = private
 
+        resolved = await self.ds.resolve_row(request)
+        pk_values = resolved.pk_values
         results = await resolved.db.execute(
             resolved.sql, resolved.params, truncate=True
         )
@@ -485,8 +504,8 @@ class RowView(BaseView):
             for row in display_rows:
                 for cell in row:
                     if cell["column"] in pk_set:
-                        cell["value"] = markupsafe.Markup(
+                        cell["value"] = markupsafe.Markup("<strong>{}</strong>").format(
-                            "<strong>{}</strong>".format(cell["value"])
+                            cell["value"]
                         )
 
             label_column = await db.label_column_for_table(table) if is_table else None
@@ -500,7 +519,7 @@ class RowView(BaseView):
 
             row_action_label = pk_path
             if row_label and row_label != pk_path:
-                row_action_label = "{} {}".format(pk_path, row_label)
+                row_action_label = f"{pk_path} {row_label}"
 
             row_action_permissions = {}
             if is_table and db.is_mutable:
@@ -513,7 +532,7 @@ class RowView(BaseView):
             row_actions = []
             if row_action_permissions.get("update-row"):
                 attrs = {
-                    "aria-label": "Edit row {}".format(row_action_label),
+                    "aria-label": f"Edit row {row_action_label}",
                     "data-row": row_path,
                     "data-row-action": "edit",
                 }
@@ -529,7 +548,7 @@ class RowView(BaseView):
                 )
             if row_action_permissions.get("delete-row"):
                 attrs = {
-                    "aria-label": "Delete row {}".format(row_action_label),
+                    "aria-label": f"Delete row {row_action_label}",
                     "data-row": row_path,
                     "data-row-action": "delete",
                 }
@@ -559,7 +578,7 @@ class RowView(BaseView):
                 "private": private,
                 "columns": reordered_columns,
                 "foreign_key_tables": await self.foreign_key_tables(
-                    database, table, pk_values
+                    database, table, pk_values, actor=request.actor
                 ),
                 "database_color": db.color,
                 "display_columns": display_columns,
@@ -636,12 +655,23 @@ class RowView(BaseView):
             ),
         )
 
-    async def foreign_key_tables(self, database, table, pk_values):
+    async def foreign_key_tables(self, database, table, pk_values, *, actor):
         if len(pk_values) != 1:
             return []
         db = self.ds.databases[database]
         all_foreign_keys = await db.get_all_foreign_keys()
-        foreign_keys = all_foreign_keys[table]["incoming"]
+        foreign_keys = []
+        table_permissions = {}
+        for fk in all_foreign_keys[table]["incoming"]:
+            other_table = fk["other_table"]
+            if other_table not in table_permissions:
+                table_permissions[other_table] = await self.ds.allowed(
+                    action="view-table",
+                    resource=TableResource(database=database, table=other_table),
+                    actor=actor,
+                )
+            if table_permissions[other_table]:
+                foreign_keys.append(fk)
         if len(foreign_keys) == 0:
             return []
 
@@ -679,7 +709,7 @@ class RowView(BaseView):
                 key,
                 ",".join(pk_values),
             )
-            foreign_key_tables.append({**fk, **{"count": count, "link": link}})
+            foreign_key_tables.append({**fk, "count": count, "link": link})
         return foreign_key_tables
 
 
@@ -698,39 +728,58 @@ def _truncated_row_flash_label(label):
     return label[: ROW_FLASH_LABEL_MAX_LENGTH - 1] + "\u2026"
 
 
-async def _row_flash_message(db, action, resolved, row=None):
+async def _row_flash_message(
+    datasette, request, action, resolved, row=None, *, refresh_row=False
+):
     pk_label = ", ".join(resolved.pk_values)
-    label_column = await db.label_column_for_table(resolved.table)
+    # Mutation permission does not grant access to stored row labels.
+    if not await datasette.allowed(
+        action="view-table",
+        resource=TableResource(database=resolved.db.name, table=resolved.table),
+        actor=request.actor,
+    ):
+        return f"{action} row {pk_label}"
+
+    if refresh_row and row is None:
+        results = await resolved.db.execute(
+            resolved.sql, resolved.params, truncate=True
+        )
+        row = results.first()
+    label_column = await resolved.db.label_column_for_table(resolved.table)
     label = row_label_from_label_column(row or resolved.row, label_column)
     if label:
         label = _truncated_row_flash_label(label)
     if label and label != pk_label:
-        return "{} row {} ({})".format(action, pk_label, label)
+        return f"{action} row {pk_label} ({label})"
-    return "{} row {}".format(action, pk_label)
+    return f"{action} row {pk_label}"
 
 
 async def _resolve_row_and_check_permission(datasette, request, permission):
-    from datasette.app import DatabaseNotFound, TableNotFound, RowNotFound
+    from datasette.app import DatabaseNotFound, RowNotFound, TableNotFound
 
     try:
-        resolved = await datasette.resolve_row(request)
+        _, _, resource = await _database_and_table_resource_from_request(
-    except DatabaseNotFound as e:
+            datasette, request
-        return False, Response.error(
-            ["Database not found: {}".format(e.database_name)], 404
         )
-    except TableNotFound as e:
+    except DatabaseNotFound as e:
-        return False, Response.error(["Table not found: {}".format(e.table)], 404)
+        return False, Response.error([f"Database not found: {e.database_name}"], 404)
-    except RowNotFound as e:
-        return False, Response.error(["Record not found: {}".format(e.pk_values)], 404)
 
-    # Ensure user has permission to delete this row
+    # Check the URL resource before resolving the row, so a denied request
+    # cannot distinguish an existing primary key from a missing one.
     if not await datasette.allowed(
         action=permission,
-        resource=TableResource(database=resolved.db.name, table=resolved.table),
+        resource=resource,
         actor=request.actor,
     ):
         return False, Response.error(["Permission denied"], 403)
 
+    try:
+        resolved = await datasette.resolve_row(request)
+    except TableNotFound as e:
+        return False, Response.error([f"Table not found: {e.table}"], 404)
+    except RowNotFound as e:
+        return False, Response.error([f"Record not found: {e.pk_values}"], 404)
+
     return True, resolved
 
 
@@ -749,11 +798,13 @@ class RowDeleteView(BaseView):
 
         # Delete table
         def delete_row(conn):
+            check_structured_write_table(conn, resolved.table)
             sqlite_utils.Database(conn)[resolved.table].delete(resolved.pk_values)
 
         try:
             await resolved.db.execute_write_fn(delete_row, request=request)
-        except Exception as e:
+        except Exception as e:  # noqa: BLE001
+            # TODO: narrow to expected write errors so Datasette bugs surface as 500s
             return Response.error([str(e)], 400)
 
         await self.ds.track_event(
@@ -769,7 +820,7 @@ class RowDeleteView(BaseView):
             table_url = self.ds.urls.table(resolved.db.name, resolved.table)
             self.ds.add_message(
                 request,
-                await _row_flash_message(resolved.db, "Deleted", resolved),
+                await _row_flash_message(self.ds, request, "Deleted", resolved),
                 self.ds.INFO,
             )
             return Response.json({"ok": True, "redirect": str(table_url)}, status=200)
@@ -793,7 +844,7 @@ class RowUpdateView(BaseView):
         try:
             data = await request.json()
         except json.JSONDecodeError as e:
-            return Response.error(["Invalid JSON: {}".format(e)])
+            return Response.error([f"Invalid JSON: {e}"])
         except PayloadTooLarge as e:
             return Response.error([str(e)], 413)
 
@@ -830,18 +881,27 @@ class RowUpdateView(BaseView):
             return Response.error(["Permission denied for alter-table"], 403)
 
         def update_row(conn):
+            check_structured_write_table(conn, resolved.table)
             sqlite_utils.Database(conn)[resolved.table].update(
                 resolved.pk_values, update, alter=alter
             )
 
         try:
             await resolved.db.execute_write_fn(update_row, request=request)
-        except Exception as e:
+        except Exception as e:  # noqa: BLE001
+            # TODO: narrow to expected write errors so Datasette bugs surface as 500s
             return Response.error([str(e)], 400)
 
         result = {"ok": True}
         returned_row = None
-        if data.get("return"):
+        # Only read back and disclose the stored row if the actor is also
+        # allowed to view this table - update-row alone must not be usable
+        # to read data the actor cannot otherwise see.
+        if data.get("return") and await self.ds.allowed(
+            action="view-table",
+            resource=TableResource(database=resolved.db.name, table=resolved.table),
+            actor=request.actor,
+        ):
             results = await resolved.db.execute(
                 resolved.sql, resolved.params, truncate=True
             )
@@ -858,16 +918,15 @@ class RowUpdateView(BaseView):
         )
 
         if request.args.get("_message"):
-            message_row = returned_row
-            if message_row is None:
-                results = await resolved.db.execute(
-                    resolved.sql, resolved.params, truncate=True
-                )
-                message_row = results.first()
             self.ds.add_message(
                 request,
                 await _row_flash_message(
-                    resolved.db, "Updated", resolved, row=message_row
+                    self.ds,
+                    request,
+                    "Updated",
+                    resolved,
+                    row=returned_row,
+                    refresh_row=True,
                 ),
                 self.ds.INFO,
             )

datasette/views/special.py

@@ -1,23 +1,25 @@
 import json
 import logging
+import secrets
+import urllib
+
+from datasette.events import CreateTokenEvent, LoginEvent, LogoutEvent
 from datasette.jump import JumpSQL, namespace_sql_params
 from datasette.plugins import pm
-from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent
 from datasette.resources import DatabaseResource, TableResource
-from datasette.utils.asgi import Response, Forbidden
 from datasette.utils import (
     UNSTABLE_API_MESSAGE,
     actor_matches_allow,
-    parse_size_limit,
     add_cors_headers,
     await_me_maybe,
     error_body,
-    tilde_encode,
+    parse_size_limit,
     tilde_decode,
+    tilde_encode,
 )
+from datasette.utils.asgi import Forbidden, Response
+
 from .base import BaseView, View
-import secrets
-import urllib
 
 logger = logging.getLogger(__name__)
 
@@ -179,9 +181,7 @@ class AutocompleteDebugView(BaseView):
                     )
                     context.update(
                         {
-                            "autocomplete_url": "{}/-/autocomplete".format(
+                            "autocomplete_url": f"{self.ds.urls.table(database_name, table_name)}/-/autocomplete",
-                                self.ds.urls.table(database_name, table_name)
-                            ),
                             "label_column": await db.label_column_for_table(table_name),
                         }
                     )
@@ -311,6 +311,7 @@ class AllowedResourcesView(BaseView):
     has_json_alternate = False
 
     async def get(self, request):
+        await self.ds.ensure_permission(action="view-instance", actor=request.actor)
         await self.ds.refresh_schemas()
 
         # Check if user has permissions-debug (to show sensitive fields)
@@ -420,8 +421,11 @@ class AllowedResourcesView(BaseView):
                     row["reason"] = resource.reasons
 
                 allowed_rows.append(row)
-        except Exception:
+        except Exception:  # noqa: BLE001
-            # If catalog tables don't exist yet, return empty results
+            # Returns empty results if the catalog tables don't exist yet, but
+            # also swallows the AttributeError raised for instance-level actions
+            # such as view-instance, which have no resource_class.
+            # TODO: handle that case explicitly and narrow this to sqlite3.Error
             return (
                 {
                     "ok": True,
@@ -523,7 +527,7 @@ class PermissionRulesView(BaseView):
 
         from datasette.utils.actions_sql import build_permission_rules_sql
 
-        union_sql, union_params, restriction_sqls = await build_permission_rules_sql(
+        union_sql, union_params, _restriction_sqls = await build_permission_rules_sql(
             self.ds, actor, action
         )
         await self.ds.refresh_schemas()
@@ -600,7 +604,7 @@ class PermissionRulesView(BaseView):
 
 
 async def _check_permission_for_actor(ds, action, parent, child, actor):
-    """Shared logic for checking permissions. Returns a dict with check results."""
+    """Shared logic for checking and explaining a permission decision."""
     if action not in ds.actions:
         return error_body(f"Unknown action: {action}", 404), 404
 
@@ -629,15 +633,28 @@ async def _check_permission_for_actor(ds, action, parent, child, actor):
 
     allowed = await ds.allowed(action=action, resource=resource_obj, actor=actor)
 
+    from datasette.utils.actions_sql import explain_permission_for_resource
+
+    explanation = await explain_permission_for_resource(
+        datasette=ds,
+        actor=actor,
+        action=action,
+        parent=parent,
+        child=child,
+    )
+
     response = {
         "ok": True,
+        "unstable": UNSTABLE_API_MESSAGE,
         "action": action,
         "allowed": bool(allowed),
+        "actor": actor,
         "resource": {
             "parent": parent,
             "child": child,
             "path": _resource_path(parent, child),
         },
+        "explanation": explanation,
     }
 
     if actor and "id" in actor:
@@ -655,11 +672,25 @@ class PermissionCheckView(BaseView):
         as_format = request.url_vars.get("format")
 
         if not as_format:
+            actions = [
+                {
+                    "name": action.name,
+                    "description": action.description,
+                    "takes_parent": action.takes_parent,
+                    "takes_child": action.takes_child,
+                    "also_requires": action.also_requires,
+                }
+                for action in sorted(
+                    self.ds.actions.values(), key=lambda action: action.name
+                )
+            ]
             return await self.render(
                 ["debug_check.html"],
                 request,
                 {
-                    "sorted_actions": sorted(self.ds.actions.keys()),
+                    "actions": actions,
+                    "actor_json": request.args.get("actor")
+                    or json.dumps(request.actor, indent=2),
                     "has_debug_permission": True,
                 },
             )
@@ -671,9 +702,18 @@ class PermissionCheckView(BaseView):
 
         parent = request.args.get("parent")
         child = request.args.get("child")
+        actor = request.actor
+        actor_json = request.args.get("actor")
+        if actor_json is not None:
+            try:
+                actor = json.loads(actor_json)
+            except json.JSONDecodeError as ex:
+                return Response.error(f"Invalid actor JSON: {ex}", 400)
+            if actor is not None and not isinstance(actor, dict):
+                return Response.error("actor must be a JSON object or null", 400)
 
         response, status = await _check_permission_for_actor(
-            self.ds, action, parent, child, request.actor
+            self.ds, action, parent, child, actor
         )
         return Response.json(response, status=status)
 
@@ -757,6 +797,8 @@ class CreateTokenView(BaseView):
             raise Forbidden(
                 "Token authentication cannot be used to create additional tokens"
             )
+        if "_r" in request.actor:
+            raise Forbidden("Restricted actors cannot create API tokens")
 
     async def shared(self, request):
         self.check_permission(request)
@@ -834,6 +876,11 @@ class CreateTokenView(BaseView):
                 else:
                     errors.append("Invalid expire duration unit")
 
+        if errors:
+            context = await self.shared(request)
+            context["errors"] = errors
+            return await self.render(["create_token.html"], request, context)
+
         # Are there any restrictions?
         from datasette.tokens import TokenRestrictions
 
@@ -900,7 +947,7 @@ class ApiExplorerView(BaseView):
                 tables.append({"name": table, "links": table_links})
                 table_links.append(
                     {
-                        "label": "Get rows for {}".format(table),
+                        "label": f"Get rows for {table}",
                         "method": "GET",
                         "path": self.ds.urls.table(name, table, format="json"),
                     }
@@ -920,7 +967,7 @@ class ApiExplorerView(BaseView):
                             {
                                 "path": self.ds.urls.table(name, table) + "/-/insert",
                                 "method": "POST",
-                                "label": "Insert rows into {}".format(table),
+                                "label": f"Insert rows into {table}",
                                 "json": {
                                     "rows": [
                                         {
@@ -934,7 +981,7 @@ class ApiExplorerView(BaseView):
                             {
                                 "path": self.ds.urls.table(name, table) + "/-/upsert",
                                 "method": "POST",
-                                "label": "Upsert rows into {}".format(table),
+                                "label": f"Upsert rows into {table}",
                                 "json": {
                                     "rows": [
                                         {
@@ -964,7 +1011,7 @@ class ApiExplorerView(BaseView):
                     table_links.append(
                         {
                             "path": self.ds.urls.table(name, table) + "/-/drop",
-                            "label": "Drop table {}".format(table),
+                            "label": f"Drop table {table}",
                             "json": {"confirm": False},
                             "method": "POST",
                         }
@@ -981,7 +1028,7 @@ class ApiExplorerView(BaseView):
                 database_links.append(
                     {
                         "path": self.ds.urls.database(name) + "/-/create",
-                        "label": "Create table in {}".format(name),
+                        "label": f"Create table in {name}",
                         "json": {
                             "table": "new_table",
                             "columns": [
@@ -1222,14 +1269,21 @@ class SchemaBaseView(BaseView):
 
     has_json_alternate = False
 
-    async def get_database_schema(self, database_name):
+    async def get_database_schema(self, database_name, actor):
         """Get schema SQL for a database."""
         db = self.ds.databases[database_name]
-        result = await db.execute(
+        allowed_tables_page = await self.ds.allowed_resources(
-            "select group_concat(sql, ';' || CHAR(10)) as schema from sqlite_master where sql is not null"
+            "view-table", actor, parent=database_name
+        )
+        allowed_table_names = {
+            resource.child async for resource in allowed_tables_page.all()
+        }
+        result = await db.execute(
+            "select tbl_name, sql from sqlite_master where sql is not null"
+        )
+        return ";\n".join(
+            row["sql"] for row in result.rows if row["tbl_name"] in allowed_table_names
         )
-        row = result.first()
-        return row["schema"] if row and row["schema"] else ""
 
     def format_json_response(self, data):
         """Format data as JSON response with CORS headers if needed."""
@@ -1291,7 +1345,7 @@ class InstanceSchemaView(SchemaBaseView):
         # Get schema for each database
         schemas = []
         for database_name in allowed_databases:
-            schema = await self.get_database_schema(database_name)
+            schema = await self.get_database_schema(database_name, request.actor)
             schemas.append({"database": database_name, "schema": schema})
 
         if format_ == "json":
@@ -1332,7 +1386,7 @@ class DatabaseSchemaView(SchemaBaseView):
         if database_name not in self.ds.databases:
             return self.format_error_response("Database not found", format_)
 
-        schema = await self.get_database_schema(database_name)
+        schema = await self.get_database_schema(database_name, request.actor)
 
         if format_ == "json":
             return self.format_json_response(
@@ -1371,7 +1425,8 @@ class TableSchemaView(SchemaBaseView):
         # Get schema for the table
         db = self.ds.databases[database_name]
         result = await db.execute(
-            "select sql from sqlite_master where name = ? and sql is not null",
+            "select sql from sqlite_master where name = ? "
+            "and type in ('table', 'view') and sql is not null",
             [table_name],
         )
         row = result.first()

datasette/views/stored_queries.py

@@ -124,7 +124,7 @@ class QueryListView(BaseView):
             pairs.append(("_next", page.next))
             next_url = self.ds.absolute_url(
                 request,
-                "{}?{}".format(request.path, urlencode(pairs)),
+                f"{request.path}?{urlencode(pairs)}",
             )
 
         current_filters = {
@@ -279,7 +279,7 @@ class QueryCreateView(BaseView):
             ),
         )
         response.status = status
-        return response
+        return _block_framing(response)
 
     async def get(self, request):
         db = await self.ds.resolve_database(request)
@@ -415,7 +415,7 @@ class QueryDefinitionView(BaseView):
         query_name = tilde_decode(request.url_vars["query"])
         query = await self.ds.get_query(db.name, query_name)
         if query is None:
-            return Response.error(["Query not found: {}".format(query_name)], 404)
+            return Response.error([f"Query not found: {query_name}"], 404)
         if not await self.ds.allowed(
             action="view-query",
             resource=QueryResource(db.name, query_name),
@@ -439,7 +439,7 @@ class QueryUpdateView(BaseView):
         query_name = tilde_decode(request.url_vars["query"])
         existing = await self.ds.get_query(db.name, query_name)
         if existing is None:
-            return Response.error(["Query not found: {}".format(query_name)], 404)
+            return Response.error([f"Query not found: {query_name}"], 404)
         if not await self.ds.allowed(
             action="update-query",
             resource=QueryResource(db.name, query_name),
@@ -527,12 +527,12 @@ class QueryEditView(BaseView):
             ),
         )
         response.status = status
-        return response
+        return _block_framing(response)
 
     async def get(self, request):
         db, query_name, existing = await self._load(request)
         if existing is None:
-            return Response.error(["Query not found: {}".format(query_name)], 404)
+            return Response.error([f"Query not found: {query_name}"], 404)
         await self.ds.ensure_permission(
             action="update-query",
             resource=QueryResource(db.name, query_name),
@@ -545,7 +545,7 @@ class QueryEditView(BaseView):
     async def post(self, request):
         db, query_name, existing = await self._load(request)
         if existing is None:
-            return Response.error(["Query not found: {}".format(query_name)], 404)
+            return Response.error([f"Query not found: {query_name}"], 404)
         if not await self.ds.allowed(
             action="update-query",
             resource=QueryResource(db.name, query_name),
@@ -629,7 +629,7 @@ class QueryDeleteView(BaseView):
     async def get(self, request):
         db, query_name, existing = await self._load(request)
         if existing is None:
-            return Response.error(["Query not found: {}".format(query_name)], 404)
+            return Response.error([f"Query not found: {query_name}"], 404)
         await self.ds.ensure_permission(
             action="delete-query",
             resource=QueryResource(db.name, query_name),
@@ -639,21 +639,23 @@ class QueryDeleteView(BaseView):
             return Response.error(
                 ["Trusted queries cannot be deleted using the API"], 403
             )
-        return await self.render(
+        return _block_framing(
-            ["query_delete.html"],
+            await self.render(
-            request,
+                ["query_delete.html"],
-            {
+                request,
-                "database": db.name,
+                {
-                "database_color": db.color,
+                    "database": db.name,
-                "query": stored_query_to_dict(existing),
+                    "database_color": db.color,
-                "query_url": self.ds.urls.table(db.name, query_name),
+                    "query": stored_query_to_dict(existing),
-            },
+                    "query_url": self.ds.urls.table(db.name, query_name),
+                },
+            )
         )
 
     async def post(self, request):
         db, query_name, existing = await self._load(request)
         if existing is None:
-            return Response.error(["Query not found: {}".format(query_name)], 404)
+            return Response.error([f"Query not found: {query_name}"], 404)
         if not await self.ds.allowed(
             action="delete-query",
             resource=QueryResource(db.name, query_name),
@@ -665,13 +667,13 @@ class QueryDeleteView(BaseView):
                 ["Trusted queries cannot be deleted using the API"], 403
             )
 
-        data, is_json = await _json_or_form_payload(request)
+        _data, is_json = await _json_or_form_payload(request)
         await self.ds.remove_query(db.name, query_name)
         if is_json:
             return Response.json({"ok": True})
         self.ds.add_message(
             request,
-            "Query “{}” deleted".format(existing.title or query_name),
+            f"Query “{existing.title or query_name}” deleted",
             self.ds.INFO,
         )
         return Response.redirect(self.ds.urls.path(self.ds.urls.database(db.name)))

datasette/views/table.py

@@ -1,50 +1,54 @@
 import asyncio
 import itertools
 import json
+import time
 import urllib
 import urllib.parse
+from dataclasses import dataclass, field
 
 import markupsafe
+import sqlite_utils
 
+from datasette import tracer
 from datasette.column_types import SQLiteType
-from datasette.extras import extra_names_from_request
+from datasette.database import QueryInterrupted
-from datasette.plugins import pm
 from datasette.events import (
     AlterTableEvent,
     DropTableEvent,
     InsertRowsEvent,
     UpsertRowsEvent,
 )
-from datasette.database import QueryInterrupted
+from datasette.extras import ExtraScope, extra_names_from_request
-from datasette import tracer
+from datasette.filters import Filters
+from datasette.plugins import pm
 from datasette.resources import DatabaseResource, TableResource
 from datasette.utils import (
-    add_cors_headers,
-    await_me_maybe,
-    call_with_supported_arguments,
     CustomJSONEncoder,
     CustomRow,
+    InvalidSql,
+    WriteJsonValueError,
+    add_cors_headers,
     append_querystring,
+    await_me_maybe,
+    call_with_supported_arguments,
     compound_keys_after_sql,
     decode_write_json_rows,
-    format_bytes,
-    make_slot_function,
-    tilde_encode,
     escape_sqlite,
     filters_should_redirect,
+    format_bytes,
     is_url,
+    make_slot_function,
     path_from_row_pks,
     path_with_added_args,
     path_with_format,
     path_with_removed_args,
     path_with_replaced_args,
+    sqlite3,
+    tilde_encode,
     to_css_class,
     truncate_url,
     urlsafe_components,
     value_as_boolean,
-    InvalidSql,
-    WriteJsonValueError,
-    sqlite3,
 )
 from datasette.utils.asgi import (
     BadRequest,
@@ -54,11 +58,8 @@ from datasette.utils.asgi import (
     Request,
     Response,
 )
-from datasette.filters import Filters
+from datasette.utils.sqlite import check_structured_write_table
-import sqlite_utils
-from dataclasses import dataclass, field
 
-from datasette.extras import ExtraScope
 from . import Context, from_extra
 from .base import BaseView, DatasetteError, stream_csv
 from .database import QueryView
@@ -536,7 +537,7 @@ async def _table_insert_ui(
         columns.append(column_data)
 
     data = {
-        "path": "{}/-/insert".format(datasette.urls.table(database_name, table_name)),
+        "path": f"{datasette.urls.table(database_name, table_name)}/-/insert",
         "tableName": table_name,
         "columns": columns,
         "bulkColumns": bulk_columns,
@@ -544,8 +545,8 @@ async def _table_insert_ui(
         "maxInsertRows": datasette.setting("max_insert_rows"),
     }
     if can_update:
-        data["upsertPath"] = "{}/-/upsert".format(
+        data["upsertPath"] = (
-            datasette.urls.table(database_name, table_name)
+            f"{datasette.urls.table(database_name, table_name)}/-/upsert"
         )
     return data
 
@@ -604,7 +605,7 @@ async def _table_alter_ui(
         columns.append(column_data)
 
     data = {
-        "path": "{}/-/alter".format(datasette.urls.table(database_name, table_name)),
+        "path": f"{datasette.urls.table(database_name, table_name)}/-/alter",
         "tableName": table_name,
         "columns": columns,
         "primaryKeys": pks,
@@ -630,9 +631,7 @@ async def _table_alter_ui(
         actor=request.actor,
     )
     if can_drop_table:
-        data["dropPath"] = "{}/-/drop".format(
+        data["dropPath"] = f"{datasette.urls.table(database_name, table_name)}/-/drop"
-            datasette.urls.table(database_name, table_name)
-        )
     return data
 
 
@@ -673,7 +672,7 @@ async def display_columns_and_rows(
     }
     pks = await db.primary_keys(table_name)
     pks_for_display = pks
-    if not pks_for_display:
+    if not pks_for_display and not await db.view_exists(table_name):
         pks_for_display = ["rowid"]
     label_column = None
     if link_column:
@@ -728,12 +727,10 @@ async def display_columns_and_rows(
             row_label = row_label_from_label_column(row, label_column)
             row_action_label = pk_path
             if row_label and row_label != pk_path:
-                row_action_label = "{} {}".format(pk_path, row_label)
+                row_action_label = f"{pk_path} {row_label}"
             table_path = datasette.urls.table(database_name, table_name)
-            row_link = '<a href="{table_path}/{flat_pks_quoted}">{flat_pks}</a>'.format(
+            row_link = (
-                table_path=table_path,
+                f'<a href="{table_path}/{row_path}">{markupsafe.escape(pk_path)!s}</a>'
-                flat_pks=str(markupsafe.escape(pk_path)),
-                flat_pks_quoted=row_path,
             )
             edit_icon = (
                 '<svg class="row-inline-action-icon" aria-hidden="true" '
@@ -760,22 +757,16 @@ async def display_columns_and_rows(
             if row_action_permissions.get("update-row"):
                 row_actions.append(
                     '<button type="button" class="row-inline-action row-inline-action-edit" '
-                    'aria-label="Edit row {row_label}" title="Edit row" '
+                    f'aria-label="Edit row {markupsafe.escape(row_action_label)}" title="Edit row" '
                     'data-row-action="edit">'
-                    "{edit_icon}</button>".format(
+                    f"{edit_icon}</button>"
-                        edit_icon=edit_icon,
-                        row_label=markupsafe.escape(row_action_label),
-                    )
                 )
             if row_action_permissions.get("delete-row"):
                 row_actions.append(
                     '<button type="button" class="row-inline-action row-inline-action-delete" '
-                    'aria-label="Delete row {row_label}" title="Delete row" '
+                    f'aria-label="Delete row {markupsafe.escape(row_action_label)}" title="Delete row" '
                     'data-row-action="delete">'
-                    "{delete_icon}</button>".format(
+                    f"{delete_icon}</button>"
-                        delete_icon=delete_icon,
-                        row_label=markupsafe.escape(row_action_label),
-                    )
                 )
             if row_actions:
                 row_link = (
@@ -843,11 +834,7 @@ async def display_columns_and_rows(
                             path_from_row_pks(row, pks, not pks),
                             column,
                         ),
-                        (
+                        (f' title="{formatted}"' if "bytes" not in formatted else ""),
-                            ' title="{}"'.format(formatted)
-                            if "bytes" not in formatted
-                            else ""
-                        ),
                         len(value),
                         "" if len(value) == 1 else "s",
                     )
@@ -916,7 +903,7 @@ async def display_columns_and_rows(
             columns = [col for col in columns if col["name"] != pks[0]]
             first_column = {
                 "name": pks[0],
-                "sortable": len(pks) == 1,
+                "sortable": pks[0] in sortable_columns,
                 "is_pk": True,
                 "type": column_details[pks[0]].type,
                 "notnull": column_details[pks[0]].notnull,
@@ -959,7 +946,7 @@ class TableInsertView(BaseView):
         try:
             data = await request.json()
         except json.JSONDecodeError as e:
-            return _errors(["Invalid JSON: {}".format(e)])
+            return _errors([f"Invalid JSON: {e}"])
         if not isinstance(data, dict):
             return _errors(["JSON must be a dictionary"])
         keys = data.keys()
@@ -987,9 +974,7 @@ class TableInsertView(BaseView):
         # Does this exceed max_insert_rows?
         max_insert_rows = self.ds.setting("max_insert_rows")
         if len(rows) > max_insert_rows:
-            return _errors(
+            return _errors([f"Too many rows, maximum allowed is {max_insert_rows}"])
-                ["Too many rows, maximum allowed is {}".format(max_insert_rows)]
-            )
 
         # Validate other parameters
         extras = {
@@ -1047,7 +1032,7 @@ class TableInsertView(BaseView):
         # Table must exist (may handle table creation in the future)
         db = self.ds.get_database(database_name)
         if not await db.table_exists(table_name):
-            return Response.error(["Table not found: {}".format(table_name)], 404)
+            return Response.error([f"Table not found: {table_name}"], 404)
 
         if upsert:
             # Must have insert-row AND upsert-row permissions
@@ -1143,6 +1128,7 @@ class TableInsertView(BaseView):
             row_pk_values_for_later = [tuple(row[pk] for pk in pks) for row in rows]
 
         def insert_or_upsert_rows(conn):
+            check_structured_write_table(conn, table_name)
             table = sqlite_utils.Database(conn)[table_name]
             kwargs = {}
             if upsert:
@@ -1170,20 +1156,36 @@ class TableInsertView(BaseView):
 
         try:
             rows = await db.execute_write_fn(insert_or_upsert_rows, request=request)
-        except Exception as e:
+        except Exception as e:  # noqa: BLE001
+            # TODO: narrow to expected write errors so Datasette bugs surface as 500s
             return Response.error([str(e)])
         result = {"ok": True}
+        # Only read back and disclose stored rows if the actor is also
+        # allowed to view this table - insert-row/update-row alone must
+        # not be usable to read data the actor cannot otherwise see.
+        if should_return and not await self.ds.allowed(
+            action="view-table",
+            resource=TableResource(database=database_name, table=table_name),
+            actor=request.actor,
+        ):
+            should_return = False
         if should_return:
             if upsert:
                 # Fetch based on initial input IDs
                 where_clause = " OR ".join(
-                    ["({})".format(" AND ".join("{} = ?".format(pk) for pk in pks))]
+                    [
+                        "({})".format(
+                            " AND ".join(f"{escape_sqlite(pk)} = ?" for pk in pks)
+                        )
+                    ]
                     * len(row_pk_values_for_later)
                 )
                 args = list(itertools.chain.from_iterable(row_pk_values_for_later))
                 fetched_rows = await db.execute(
-                    "select {}* from [{}] where {}".format(
+                    "select {}* from {} where {}".format(
-                        "rowid, " if pks == ["rowid"] else "", table_name, where_clause
+                        "rowid, " if pks == ["rowid"] else "",
+                        escape_sqlite(table_name),
+                        where_clause,
                     ),
                     args,
                 )
@@ -1267,7 +1269,7 @@ class TableSetColumnTypeView(BaseView):
         try:
             data = await request.json()
         except json.JSONDecodeError as e:
-            return Response.error(["Invalid JSON: {}".format(e)], 400)
+            return Response.error([f"Invalid JSON: {e}"], 400)
         except PayloadTooLarge as e:
             return Response.error([str(e)], 413)
 
@@ -1294,7 +1296,7 @@ class TableSetColumnTypeView(BaseView):
             database_name, table_name
         )
         if column not in column_details:
-            return Response.error(["Column not found: {}".format(column)], 400)
+            return Response.error([f"Column not found: {column}"], 400)
 
         column_type_data = data["column_type"]
         if column_type_data is None:
@@ -1335,7 +1337,7 @@ class TableSetColumnTypeView(BaseView):
             return Response.error(['"column_type.config" must be a dictionary'], 400)
 
         if column_type not in self.ds._column_types:
-            return Response.error(["Unknown column type: {}".format(column_type)], 400)
+            return Response.error([f"Unknown column type: {column_type}"], 400)
 
         try:
             await self.ds.set_column_type(
@@ -1373,7 +1375,7 @@ class TableDropView(BaseView):
         # Table must exist
         db = self.ds.get_database(database_name)
         if not await db.table_exists(table_name):
-            return Response.error(["Table not found: {}".format(table_name)], 404)
+            return Response.error([f"Table not found: {table_name}"], 404)
         if not await self.ds.allowed(
             action="drop-table",
             resource=TableResource(database=database_name, table=table_name),
@@ -1398,7 +1400,9 @@ class TableDropView(BaseView):
                     "database": database_name,
                     "table": table_name,
                     "row_count": (
-                        await db.execute("select count(*) from [{}]".format(table_name))
+                        await db.execute(
+                            f"select count(*) from {escape_sqlite(table_name)}"
+                        )
                     ).single_value(),
                     "message": 'Pass "confirm": true to confirm',
                 },
@@ -1407,7 +1411,9 @@ class TableDropView(BaseView):
 
         # Drop table
         def drop_table(conn):
-            sqlite_utils.Database(conn)[table_name].drop()
+            table = sqlite_utils.Database(conn)[table_name]
+            table.disable_fts()
+            table.drop()
 
         await db.execute_write_fn(drop_table, request=request)
         await self.ds.track_event(
@@ -1417,12 +1423,48 @@ class TableDropView(BaseView):
         )
         self.ds.add_message(
             request,
-            "Table {} dropped".format(table_name),
+            f"Table {table_name} dropped",
             self.ds.WARNING,
         )
         return Response.json({"ok": True}, status=200)
 
 
+class TableCountView(BaseView):
+    name = "table-count"
+
+    async def post(self, request):
+        try:
+            return await self.count(request)
+        except (NotFound, Forbidden, BadRequest, DatasetteError) as ex:
+            return Response.error(str(ex), status=ex.status)
+
+    async def count(self, request):
+        resolved = await self.ds.resolve_table(request)
+        visible, _private = await self.ds.check_visibility(
+            request.actor,
+            action="view-table",
+            resource=TableResource(database=resolved.db.name, table=resolved.table),
+        )
+        if not visible:
+            raise Forbidden("You do not have permission to view this table")
+        _, where_clauses, params, _, _ = await _table_filters(
+            self.ds, request, resolved.db.name, resolved.table
+        )
+        sql = f"select count(*) from {escape_sqlite(resolved.table)}"
+        if where_clauses:
+            sql += " where " + " and ".join(where_clauses)
+        try:
+            results = await resolved.db.execute(sql, params)
+        except QueryInterrupted:
+            return Response.error("Count query timed out", status=400)
+        except (sqlite3.OperationalError, InvalidSql) as ex:
+            return Response.error(str(ex), status=400)
+        return Response.json(
+            {"ok": True, "count": results.single_value()},
+            headers={"Cache-Control": "no-store"},
+        )
+
+
 class TableFragmentView(BaseView):
     name = "table-fragment"
 
@@ -1477,32 +1519,28 @@ def _prefix_range_end(value):
 
 
 def _autocomplete_like(column):
-    return "{} like :like escape char(92)".format(escape_sqlite(column))
+    return f"{escape_sqlite(column)} like :like escape char(92)"
 
 
 def _autocomplete_prefix_like(column):
-    return "{} like :prefix escape char(92)".format(escape_sqlite(column))
+    return f"{escape_sqlite(column)} like :prefix escape char(92)"
 
 
 def _autocomplete_order_by(pks, label_column, exact_pk, label_matches_first=True):
     clauses = []
     if exact_pk:
         clauses.append(
-            "case when cast({} as text) = :q then 0 else 1 end".format(
+            f"case when cast({escape_sqlite(pks[0])} as text) = :q then 0 else 1 end"
-                escape_sqlite(pks[0])
-            )
         )
     if label_column:
         label_like = _autocomplete_like(label_column)
         if label_matches_first:
-            clauses.append("case when {} then 0 else 1 end".format(label_like))
+            clauses.append(f"case when {label_like} then 0 else 1 end")
         clauses.append(
-            "case when {} then length(cast({} as text)) end".format(
+            f"case when {label_like} then length(cast({escape_sqlite(label_column)} as text)) end"
-                label_like, escape_sqlite(label_column)
-            )
         )
     else:
-        clauses.append("length(cast({} as text))".format(escape_sqlite(pks[0])))
+        clauses.append(f"length(cast({escape_sqlite(pks[0])} as text))")
     clauses.extend(escape_sqlite(pk) for pk in pks)
     return ", ".join(clauses)
 
@@ -1569,8 +1607,8 @@ class TableAutocompleteView(BaseView):
             return Response.json({"ok": True, "rows": []})
         params = {
             "q": q,
-            "like": "%{}%".format(_escape_like(q)),
+            "like": f"%{_escape_like(q)}%",
-            "prefix": "{}%".format(_escape_like(q)),
+            "prefix": f"{_escape_like(q)}%",
         }
 
         like_columns = pks[:]
@@ -1584,18 +1622,13 @@ class TableAutocompleteView(BaseView):
             where_sql = "1 = 1"
             order_by = _autocomplete_initial_order_by(pks)
 
-        sql = """
+        sql = f"""
             select {select_sql}
-            from {table}
+            from {escape_sqlite(table_name)}
-            where {where}
+            where {where_sql}
             order by {order_by}
             limit 10
-        """.format(
+        """
-            select_sql=select_sql,
-            table=escape_sqlite(table_name),
-            where=where_sql,
-            order_by=order_by,
-        )
 
         try:
             results = await db.execute(
@@ -1607,21 +1640,14 @@ class TableAutocompleteView(BaseView):
             if prefix_end:
                 params["prefix_end"] = prefix_end
                 first_pk = escape_sqlite(pks[0])
-                fallback_where = (
+                fallback_where = f"{first_pk} >= :q and {first_pk} < :prefix_end and {fallback_where}"
-                    "{first_pk} >= :q and {first_pk} < :prefix_end and {like}"
+            fallback_sql = f"""
-                ).format(first_pk=first_pk, like=fallback_where)
-            fallback_sql = """
                 select {select_sql}
-                from {table}
+                from {escape_sqlite(table_name)}
-                where {where}
+                where {fallback_where}
-                order by {order_by}
+                order by {_autocomplete_pk_order_by(pks)}
                 limit 10
-            """.format(
+            """
-                select_sql=select_sql,
-                table=escape_sqlite(table_name),
-                where=fallback_where,
-                order_by=_autocomplete_pk_order_by(pks),
-            )
             try:
                 results = await db.execute(
                     fallback_sql,
@@ -1725,13 +1751,22 @@ async def table_view(datasette, request):
     if ttl is None or not ttl.isdigit():
         ttl = datasette.setting("default_cache_ttl")
 
+    private = getattr(request, "_datasette_private_response", False)
+
     if datasette.cache_headers and response.status == 200:
-        ttl = int(ttl)
+        if private:
-        if ttl == 0:
+            # This response is only visible to the current actor (denied to
-            ttl_header = "no-cache"
+            # anonymous requests), so it must never be stored by a shared
+            # cache/CDN - and ?_ttl= must not be able to override that.
+            response.headers["Cache-Control"] = "private, no-store"
+            response.headers["Vary"] = "Cookie"
         else:
-            ttl_header = f"max-age={ttl}"
+            ttl = int(ttl)
-        response.headers["Cache-Control"] = ttl_header
+            if ttl == 0:
+                ttl_header = "no-cache"
+            else:
+                ttl_header = f"max-age={ttl}"
+            response.headers["Cache-Control"] = ttl_header
 
     # Referrer policy
     response.headers["Referrer-Policy"] = "no-referrer"
@@ -1767,6 +1802,7 @@ async def table_view_traced(datasette, request):
         context_for_html_hack = True
         default_labels = True
 
+    start = time.perf_counter()
     view_data = await table_view_data(
         datasette,
         request,
@@ -1777,7 +1813,8 @@ async def table_view_traced(datasette, request):
     )
     if isinstance(view_data, Response):
         return view_data
-    data, rows, columns, expanded_columns, sql, next_url = view_data
+    query_ms = (time.perf_counter() - start) * 1000
+    data, rows, columns, _expanded_columns, sql, next_url = view_data
 
     # Handle formats from plugins
     if format_ == "csv":
@@ -1788,8 +1825,8 @@ async def table_view_traced(datasette, request):
                 rows,
                 columns,
                 expanded_columns,
-                sql,
+                _sql,
-                next_url,
+                _next_url,
             ) = await table_view_data(
                 datasette,
                 request,
@@ -1806,7 +1843,7 @@ async def table_view_traced(datasette, request):
             return data, None, None
 
         return await stream_csv(datasette, fetch_data, request, resolved.db.name)
-    elif format_ in datasette.renderers.keys():
+    elif format_ in datasette.renderers:
         # Dispatch request to the correct output format renderer
         # (CSV is not handled here due to streaming)
         result = call_with_supported_arguments(
@@ -1864,9 +1901,7 @@ async def table_view_traced(datasette, request):
         )
         headers.update(
             {
-                "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format(
+                "Link": f'<{alternate_url_json}>; rel="alternate"; type="application/json+datasette"'
-                    alternate_url_json
-                )
             }
         )
         table_context = TableContext(
@@ -1925,7 +1960,7 @@ async def table_view_traced(datasette, request):
                 resource=DatabaseResource(database=resolved.db.name),
                 actor=request.actor,
             ),
-            query_ms=1.2,
+            query_ms=query_ms,
             select_templates=[
                 f"{'*' if template_name == template.name else ''}{template_name}"
                 for template_name in templates
@@ -1951,12 +1986,53 @@ async def table_view_traced(datasette, request):
             headers=headers,
         )
     else:
-        assert False, "Invalid format: {}".format(format_)
+        assert False, f"Invalid format: {format_}"
     if next_url:
         r.headers["link"] = f'<{next_url}>; rel="next"'
     return r
 
 
+async def _table_filters(datasette, request, database_name, table_name):
+    # Arguments that start with _ and don't contain a __ are
+    # special - things like ?_search= - and should not be
+    # treated as filters.
+    filter_args = []
+    for key in request.args:
+        if not (key.startswith("_") and "__" not in key):
+            for v in request.args.getlist(key):
+                filter_args.append((key, v))
+
+    # Build where clauses from query string arguments
+    filters = Filters(sorted(filter_args))
+    where_clauses, params = filters.build_where_clauses(table_name)
+
+    # Execute filters_from_request plugin hooks - including the default
+    # ones that live in datasette/filters.py
+    extra_context_from_filters = {}
+    extra_human_descriptions = []
+
+    for hook in pm.hook.filters_from_request(
+        request=request,
+        table=table_name,
+        database=database_name,
+        datasette=datasette,
+    ):
+        filter_arguments = await await_me_maybe(hook)
+        if filter_arguments:
+            where_clauses.extend(filter_arguments.where_clauses)
+            params.update(filter_arguments.params)
+            extra_human_descriptions.extend(filter_arguments.human_descriptions)
+            extra_context_from_filters.update(filter_arguments.extra_context)
+
+    return (
+        filters,
+        where_clauses,
+        params,
+        extra_human_descriptions,
+        extra_context_from_filters,
+    )
+
+
 async def table_view_data(
     datasette,
     request,
@@ -1981,6 +2057,10 @@ async def table_view_data(
     )
     if not visible:
         raise Forbidden("You do not have permission to view this table")
+    # Record whether this response is private (visible to this actor only)
+    # so the outer table_view() can set appropriate Cache-Control headers,
+    # regardless of which output format ends up being rendered.
+    request._datasette_private_response = private
 
     # Redirect based on request.args, if necessary
     redirect_response = await _redirect_if_needed(datasette, request, resolved)
@@ -2031,36 +2111,13 @@ async def table_view_data(
 
     table_metadata = await datasette.table_config(database_name, table_name)
 
-    # Arguments that start with _ and don't contain a __ are
+    (
-    # special - things like ?_search= - and should not be
+        filters,
-    # treated as filters.
+        where_clauses,
-    filter_args = []
+        params,
-    for key in request.args:
+        extra_human_descriptions,
-        if not (key.startswith("_") and "__" not in key):
+        extra_context_from_filters,
-            for v in request.args.getlist(key):
+    ) = await _table_filters(datasette, request, database_name, table_name)
-                filter_args.append((key, v))
-
-    # Build where clauses from query string arguments
-    filters = Filters(sorted(filter_args))
-    where_clauses, params = filters.build_where_clauses(table_name)
-
-    # Execute filters_from_request plugin hooks - including the default
-    # ones that live in datasette/filters.py
-    extra_context_from_filters = {}
-    extra_human_descriptions = []
-
-    for hook in pm.hook.filters_from_request(
-        request=request,
-        table=table_name,
-        database=database_name,
-        datasette=datasette,
-    ):
-        filter_arguments = await await_me_maybe(hook)
-        if filter_arguments:
-            where_clauses.extend(filter_arguments.where_clauses)
-            params.update(filter_arguments.params)
-            extra_human_descriptions.extend(filter_arguments.human_descriptions)
-            extra_context_from_filters.update(filter_arguments.extra_context)
 
     # Deal with custom sort orders
     sortable_columns = await _sortable_columns_for_table(
@@ -2142,9 +2199,7 @@ async def table_view_data(
                             extra_desc_only=(
                                 ""
                                 if sort
-                                else " or {column2} is null".format(
+                                else f" or {escape_sqlite(sort or sort_desc)} is null"
-                                    column2=escape_sqlite(sort or sort_desc)
-                                )
                             ),
                             next_clauses=" and ".join(next_by_pk_clauses),
                         )
@@ -2186,22 +2241,11 @@ async def table_view_data(
 
     # Facets are calculated against SQL without order by or limit
     sql_no_order_no_limit = (
-        "select {select_all_columns} from {table_name} {where}".format(
+        f"select {select_all_columns} from {escape_sqlite(table_name)} {where_clause}"
-            select_all_columns=select_all_columns,
-            table_name=escape_sqlite(table_name),
-            where=where_clause,
-        )
     )
 
     # This is the SQL that populates the main table on the page
-    sql = "select {select_specified_columns} from {table_name} {where}{order_by} limit {page_size}{offset}".format(
+    sql = f"select {select_specified_columns} from {escape_sqlite(table_name)} {where_clause}{order_by} limit {page_size + 1}{offset}"
-        select_specified_columns=select_specified_columns,
-        table_name=escape_sqlite(table_name),
-        where=where_clause,
-        order_by=order_by,
-        page_size=page_size + 1,
-        offset=offset,
-    )
 
     if request.args.get("_timelimit"):
         extra_args["custom_time_limit"] = int(request.args.get("_timelimit"))
@@ -2212,9 +2256,6 @@ async def table_view_data(
     except (sqlite3.OperationalError, InvalidSql) as e:
         raise DatasetteError(str(e), title="Invalid SQL", status=400)
 
-    except sqlite3.OperationalError as e:
-        raise DatasetteError(str(e))
-
     columns = [r[0] for r in results.description]
     rows = list(results.rows)
 
@@ -2261,7 +2302,8 @@ async def table_view_data(
             new_rows = []
             for row in rows:
                 new_row = CustomRow(columns)
-                for column in row.keys():
+                # CustomRow/sqlite3.Row iterate over values, so .keys() is required
+                for column in row.keys():  # noqa: SIM118
                     value = row[column]
                     if (column, value) in expanded_labels and value is not None:
                         new_row[column] = {
@@ -2273,8 +2315,6 @@ async def table_view_data(
                 new_rows.append(new_row)
             rows = new_rows
 
-    _next = request.args.get("_next")
-
     # Pagination next link
     next_value, next_url = await _next_value_and_url(
         datasette,
@@ -2298,7 +2338,7 @@ async def table_view_data(
         # Data formats reject unknown extras; the HTML path (which passes
         # extra_extras={"_html"}) resolves internal extras of its own
         table_extra_registry.validate_requested(extras, ExtraScope.TABLE)
-    if any(k for k in request.args.keys() if k == "_facet" or k.startswith("_facet_")):
+    if any(k for k in request.args if k == "_facet" or k.startswith("_facet_")):
         extras.add("facet_results")
     if request.args.get("_shape") == "object":
         extras.add("primary_keys")
@@ -2479,20 +2519,16 @@ async def _next_value_and_url(
             except IndexError:
                 # sort/sort_desc column missing from SELECT - look up value by PK instead
                 prefix_where_clause = " and ".join(
-                    "[{}] = :pk{}".format(pk, i) for i, pk in enumerate(pks)
+                    f"{escape_sqlite(pk)} = :pk{i}" for i, pk in enumerate(pks)
                 )
-                prefix_lookup_sql = "select [{}] from [{}] where {}".format(
+                prefix_lookup_sql = (
-                    sort or sort_desc, table_name, prefix_where_clause
+                    f"select {escape_sqlite(sort or sort_desc)} "
+                    f"from {escape_sqlite(table_name)} where {prefix_where_clause}"
                 )
                 prefix = (
                     await db.execute(
                         prefix_lookup_sql,
-                        {
+                        {**{f"pk{i}": rows[-2][pk] for i, pk in enumerate(pks)}},
-                            **{
-                                "pk{}".format(i): rows[-2][pk]
-                                for i, pk in enumerate(pks)
-                            }
-                        },
                     )
                 ).single_value()
             if isinstance(prefix, dict) and "value" in prefix:

datasette/views/table_create_alter.py

@@ -1,9 +1,9 @@
 import json
 import re
 import time
-from typing import Annotated, Any, Literal, Union
+from typing import Annotated, Any, Literal
 
-from datasette.database import QueryInterrupted
+import sqlite_utils
 from pydantic import (
     BaseModel,
     ConfigDict,
@@ -13,21 +13,29 @@ from pydantic import (
     model_validator,
 )
 from pydantic_core import PydanticCustomError
-import sqlite_utils
 from sqlite_utils.db import DEFAULT as SQLITE_UTILS_DEFAULT
 
 from datasette.column_types import SQLiteType
+from datasette.database import QueryInterrupted
 from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent
 from datasette.resources import DatabaseResource, TableResource
 from datasette.utils import (
+    WriteJsonValueError,
     decode_write_json_rows,
     escape_sqlite,
     get_outbound_foreign_keys,
     table_column_details,
-    WriteJsonValueError,
 )
 from datasette.utils.asgi import NotFound, PayloadTooLarge, Response
-from datasette.utils.sqlite import sqlite_hidden_table_names
+from datasette.utils.permissions import (
+    SKIP_PERMISSION_CHECKS,
+    gather_permission_sql_from_hooks,
+    resolve_permissions_with_candidates,
+)
+from datasette.utils.sqlite import (
+    check_structured_write_table,
+    sqlite_hidden_table_names,
+)
 
 from .base import BaseView
 
@@ -122,6 +130,30 @@ def _public_foreign_key_target(target):
     }
 
 
+async def _filter_visible_foreign_key_targets(datasette, actor, database_name, targets):
+    if not targets:
+        return []
+
+    permission_sqls = await gather_permission_sql_from_hooks(
+        datasette=datasette,
+        actor=actor,
+        action="view-table",
+    )
+    if permission_sqls is SKIP_PERMISSION_CHECKS:
+        return targets
+
+    candidate_tables = list(dict.fromkeys(target["fk_table"] for target in targets))
+    permission_rows = await resolve_permissions_with_candidates(
+        datasette.get_internal_database(),
+        actor,
+        permission_sqls,
+        [(database_name, table_name) for table_name in candidate_tables],
+        "view-table",
+    )
+    visible_tables = {row["child"] for row in permission_rows if bool(row["allow"])}
+    return [target for target in targets if target["fk_table"] in visible_tables]
+
+
 def _singular(name):
     if name.endswith("ies") and len(name) > 3:
         return name[:-3] + "y"
@@ -136,14 +168,14 @@ def _foreign_key_name_reasons(source_column, target):
     singular_table = _singular(table)
     column = target["fk_column"].lower()
     possible_names = {
-        "{}_{}".format(table, column),
+        f"{table}_{column}",
-        "{}_{}".format(singular_table, column),
+        f"{singular_table}_{column}",
     }
     if column == "id":
         possible_names.update(
             {
-                "{}_id".format(table),
+                f"{table}_id",
-                "{}_id".format(singular_table),
+                f"{singular_table}_id",
             }
         )
     return ["name_match"] if source in possible_names else []
@@ -262,10 +294,8 @@ async def _create_table_ui_context(
     if not database_action_permissions.get("create-table"):
         return None
     data = {
-        "path": "{}/-/create".format(datasette.urls.database(database_name)),
+        "path": f"{datasette.urls.database(database_name)}/-/create",
-        "foreignKeyTargetsPath": "{}/-/foreign-key-targets".format(
+        "foreignKeyTargetsPath": f"{datasette.urls.database(database_name)}/-/foreign-key-targets",
-            datasette.urls.database(database_name)
-        ),
         "databaseName": database_name,
         "columnTypes": CREATE_TABLE_COLUMN_TYPES,
         "defaultExpressions": default_expression_options(),
@@ -398,15 +428,15 @@ def default_expr_for_sql(expression):
 
 def _quoted_options(options):
     if len(options) == 1:
-        return "'{}'".format(options[0])
+        return f"'{options[0]}'"
     return "{} or '{}'".format(
-        ", ".join("'{}'".format(option) for option in options[:-1]),
+        ", ".join(f"'{option}'" for option in options[:-1]),
         options[-1],
     )
 
 
 def _default_expr_error_message():
-    return "Input should be {}".format(_quoted_options(list(DEFAULT_EXPRESSIONS)))
+    return f"Input should be {_quoted_options(list(DEFAULT_EXPRESSIONS))}"
 
 
 def default_expression_options():
@@ -715,18 +745,16 @@ class SetForeignKeysOperation(_StrictPydanticModel):
 
 
 AlterTableOperation = Annotated[
-    Union[
+    AddColumnOperation
-        AddColumnOperation,
+    | RenameColumnOperation
-        RenameColumnOperation,
+    | RenameTableOperation
-        RenameTableOperation,
+    | AlterColumnOperation
-        AlterColumnOperation,
+    | DropColumnOperation
-        DropColumnOperation,
+    | SetPrimaryKeyOperation
-        SetPrimaryKeyOperation,
+    | ReorderColumnsOperation
-        ReorderColumnsOperation,
+    | AddForeignKeyOperation
-        AddForeignKeyOperation,
+    | DropForeignKeyOperation
-        DropForeignKeyOperation,
+    | SetForeignKeysOperation,
-        SetForeignKeysOperation,
-    ],
     Field(discriminator="op"),
 ]
 
@@ -740,7 +768,7 @@ def _pydantic_errors(validation_error):
     for error in validation_error.errors():
         location = ".".join(str(item) for item in error["loc"])
         message = error["msg"]
-        errors.append("{}: {}".format(location, message) if location else message)
+        errors.append(f"{location}: {message}" if location else message)
     return errors
 
 
@@ -761,7 +789,7 @@ def _create_table_pydantic_errors(validation_error):
             output.append(message)
             continue
         location = ".".join(str(item) for item in error["loc"])
-        output.append("{}: {}".format(location, message) if location else message)
+        output.append(f"{location}: {message}" if location else message)
     return output
 
 
@@ -810,7 +838,7 @@ class TableCreateView(BaseView):
         try:
             data = await request.json()
         except json.JSONDecodeError as e:
-            return Response.error(["Invalid JSON: {}".format(e)])
+            return Response.error([f"Invalid JSON: {e}"])
         except PayloadTooLarge as e:
             return Response.error([str(e)], 413)
 
@@ -825,17 +853,18 @@ class TableCreateView(BaseView):
         ignore = create_request.ignore
         replace = create_request.replace
 
-        if replace:
-            # Must have update-row permission
-            if not await self.ds.allowed(
-                action="update-row",
-                resource=DatabaseResource(database=database_name),
-                actor=request.actor,
-            ):
-                return Response.error(["Permission denied: need update-row"], 403)
-
         table_name = create_request.table
         table_exists = await db.table_exists(table_name)
+        table_resource = TableResource(database=database_name, table=table_name)
+
+        # Replacing rows requires update-row permission
+        if replace and not await self.ds.allowed(
+            action="update-row",
+            resource=table_resource,
+            actor=request.actor,
+        ):
+            return Response.error(["Permission denied: need update-row"], 403)
+
         columns = create_request.columns
         rows = create_request.rows_list
 
@@ -843,7 +872,7 @@ class TableCreateView(BaseView):
             # Must have insert-row permission
             if not await self.ds.allowed(
                 action="insert-row",
-                resource=DatabaseResource(database=database_name),
+                resource=table_resource,
                 actor=request.actor,
             ):
                 return Response.error(["Permission denied: need insert-row"], 403)
@@ -862,7 +891,7 @@ class TableCreateView(BaseView):
                 if create_request.alter:
                     if not await self.ds.allowed(
                         action="alter-table",
-                        resource=DatabaseResource(database=database_name),
+                        resource=table_resource,
                         actor=request.actor,
                     ):
                         return Response.error(
@@ -878,9 +907,14 @@ class TableCreateView(BaseView):
             actual_pks = await db.primary_keys(table_name)
             # if pk passed and table already exists check it does not change
             bad_pks = False
-            if len(actual_pks) == 1 and pk and pk != actual_pks[0]:
+            if (
-                bad_pks = True
+                len(actual_pks) == 1
-            elif len(actual_pks) > 1 and pks and set(pks) != set(actual_pks):
+                and pk
+                and pk != actual_pks[0]
+                or len(actual_pks) > 1
+                and pks
+                and set(pks) != set(actual_pks)
+            ):
                 bad_pks = True
             if bad_pks:
                 return Response.error(["pk cannot be changed for existing table"])
@@ -893,6 +927,7 @@ class TableCreateView(BaseView):
             )
 
         def create_table(conn):
+            check_structured_write_table(conn, table_name, allow_missing=True)
             db_for_write = sqlite_utils.Database(conn)
             table = db_for_write[table_name]
             if rows:
@@ -925,7 +960,8 @@ class TableCreateView(BaseView):
 
         try:
             schema = await db.execute_write_fn(create_table, request=request)
-        except Exception as e:
+        except Exception as e:  # noqa: BLE001
+            # TODO: narrow to expected write errors so Datasette bugs surface as 500s
             return Response.error([str(e)])
 
         if initial_schema is not None and initial_schema != schema:
@@ -1011,6 +1047,9 @@ class DatabaseForeignKeyTargetsView(BaseView):
             for target in (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts()
             if target["fk_table"] not in hidden_tables
         ]
+        targets = await _filter_visible_foreign_key_targets(
+            self.ds, request.actor, database_name, targets
+        )
         return Response.json(
             {
                 "ok": True,
@@ -1049,6 +1088,15 @@ class TableForeignKeySuggestionsView(BaseView):
         source_columns, targets, current_by_column = await db.execute_fn(
             lambda conn: _foreign_key_suggestion_metadata(conn, table_name)
         )
+        targets = await _filter_visible_foreign_key_targets(
+            self.ds, request.actor, database_name, targets
+        )
+        visible_target_tables = {target["fk_table"] for target in targets}
+        current_by_column = {
+            column: current
+            for column, current in current_by_column.items()
+            if current["fk_table"] in visible_target_tables
+        }
 
         columns = []
         options_by_column = {}
@@ -1171,7 +1219,7 @@ class TableAlterView(BaseView):
         try:
             data = await request.json()
         except json.JSONDecodeError as e:
-            return Response.error(["Invalid JSON: {}".format(e)], 400)
+            return Response.error([f"Invalid JSON: {e}"], 400)
         except PayloadTooLarge as e:
             return Response.error([str(e)], 413)
 
@@ -1261,7 +1309,9 @@ class TableAlterView(BaseView):
                     elif operation.op == "set_foreign_keys":
                         foreign_keys = [fk.tuple for fk in args.foreign_keys]
 
-                with operation_conn:
+                # Use a savepoint inside execute_write_fn's transaction so
+                # write_wrapper hooks can still reject and roll back the write.
+                with db_for_write.atomic():
                     for column in add_columns:
                         not_null_default = None
                         if column.not_null:
@@ -1311,10 +1361,7 @@ class TableAlterView(BaseView):
                         and rename_table_to != current_table_name
                     ):
                         operation_conn.execute(
-                            "alter table {} rename to {}".format(
+                            f"alter table {escape_sqlite(current_table_name)} rename to {escape_sqlite(rename_table_to)}"
-                                escape_sqlite(current_table_name),
-                                escape_sqlite(rename_table_to),
-                            )
                         )
                         current_table_name = rename_table_to
 
@@ -1329,7 +1376,8 @@ class TableAlterView(BaseView):
             before_schema, after_schema, after_table_name = await db.execute_write_fn(
                 alter_table, request=request
             )
-        except Exception as e:
+        except Exception as e:  # noqa: BLE001
+            # TODO: narrow to expected write errors so Datasette bugs surface as 500s
             return Response.error([str(e)], 400)
 
         altered = before_schema != after_schema