Extract reusable modal dialogs as a documented JS utility

Merge pull request #2948 from simonw/codex/reusable-modal-dialogs

Closes #2790

.github/workflows/deploy-latest.yml

@@ -14,24 +14,46 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
+    - name: Check deployment prerequisites
+      id: deployment-prerequisites
+      env:
+        GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+        LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
+      run: |
+        missing=()
+        for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do
+          if [[ -z "${!variable:-}" ]]; then
+            missing+=("$variable")
+          fi
+        done
+        if (( ${#missing[@]} )); then
+          echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}"
+          echo "available=false" >> "$GITHUB_OUTPUT"
+        else
+          echo "available=true" >> "$GITHUB_OUTPUT"
+        fi
     - name: Check out datasette
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/checkout@v7
     - name: Set up Python
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/setup-python@v6
       with:
         python-version: "3.13"
         cache: pip
     - name: Install Python dependencies
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         python -m pip install --upgrade pip
         python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1
+        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
     - name: Run tests
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
     - name: Build fixtures.db and other files needed to deploy the demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |-
         python tests/fixtures.py \
           fixtures.db \
@@ -39,14 +61,18 @@ jobs:
           fixtures-metadata.json \
           plugins \
           --extra-db-filename extra_database.db
+        # Package the config with the plugins, excluding test-only plugin secrets
+        # that reference temporary files outside the deployed container.
+        jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json
     - name: Build docs.db
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
     - name: Set up the alternate-route demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         echo '
         from datasette import hookimpl
@@ -58,6 +84,7 @@ jobs:
         ' > plugins/alternative_route.py
         cp fixtures.db fixtures2.db
     - name: And the counters writable stored query demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         cat > plugins/counters.py <<EOF
         from datasette import hookimpl
@@ -97,12 +124,15 @@ jobs:
     #     cat metadata.json
     - id: auth
       name: Authenticate to Google Cloud
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/auth@v3
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}
     - name: Set up Cloud SDK
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/setup-gcloud@v3
     - name: Deploy to Cloud Run
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       env:
         LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
       run: |-
@@ -117,16 +147,16 @@ jobs:
             --plugins-dir=plugins \
             --branch=$GITHUB_SHA \
             --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
+            --extra-options="--config plugins/fixtures-config.json --setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
             --install 'datasette-ephemeral-tables>=0.2.2' \
             --service "datasette-latest$SUFFIX" \
             --secret $LATEST_DATASETTE_SECRET
-    - name: Deploy to docs as well (only for main)
-      if: ${{ github.ref == 'refs/heads/main' }}
+    - name: Upload latest documentation database to S3 (only for main)
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
       run: |-
-        # Deploy docs.db to a different service
-        datasette publish cloudrun docs.db \
-            --branch=$GITHUB_SHA \
-            --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1" \
-            --service=datasette-docs-latest
+        # Keep development documentation separate from the stable release database.
+        s3-credentials put-object datasette-docs latest/docs.db docs.db \
+          --content-type application/octet-stream

.github/workflows/playwright.yml

@@ -2,9 +2,15 @@ name: Playwright
 
 on:
   push:
+    branches:
+    - main
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 permissions:
   contents: read
 

.github/workflows/prettier.yml

@@ -1,6 +1,15 @@
 name: Check JavaScript for conformance with Prettier
 
-on: [push]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read

.github/workflows/publish.yml

@@ -2,7 +2,7 @@ name: Publish Python Package
 
 on:
   release:
-    types: [created]
+    types: [published]
 
 permissions:
   contents: read
@@ -51,6 +51,8 @@ jobs:
     - name: Publish
       uses: pypa/gh-action-pypi-publish@release/v1
 
+  # After the first non-prerelease 1.0 release, disable this job on 0.65.x,
+  # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs.
   deploy_static_docs:
     runs-on: ubuntu-latest
     needs: [deploy]
@@ -66,26 +68,20 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1
+        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
     - name: Build docs.db
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
-    - id: auth
-      name: Authenticate to Google Cloud
-      uses: google-github-actions/auth@v2
-      with:
-        credentials_json: ${{ secrets.GCP_SA_KEY }}
-    - name: Set up Cloud SDK
-      uses: google-github-actions/setup-gcloud@v3
-    - name: Deploy stable-docs.datasette.io to Cloud Run
+    - name: Upload stable documentation database to S3
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
       run: |-
-        gcloud config set run/region us-central1
-        gcloud config set project datasette-222320
-        datasette publish cloudrun docs.db \
-            --service=datasette-docs-stable
+        s3-credentials put-object datasette-docs docs.db docs.db \
+          --content-type application/octet-stream
 
   deploy_docker:
     runs-on: ubuntu-latest

.github/workflows/spellcheck.yml

@@ -1,6 +1,15 @@
 name: Check spelling in documentation
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read

.github/workflows/test-coverage.yml

@@ -1,40 +0,0 @@
-name: Calculate test coverage
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out datasette
-      uses: actions/checkout@v7
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version: '3.12'
-        cache: 'pip'
-        cache-dependency-path: '**/pyproject.toml'
-    - name: Install Python dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install . --group dev
-        python -m pip install pytest-cov
-    - name: Run tests
-      run: |-
-        ls -lah
-        cat .coveragerc
-        pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x
-        ls -lah
-    - name: Upload coverage report
-      uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.xml

.github/workflows/test-pyodide.yml

@@ -2,9 +2,15 @@ name: Test in Pyodide with shot-scraper
 
 on:
   push:
+    branches:
+    - main
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 permissions:
   contents: read
 

.github/workflows/test-sqlite-support.yml

@@ -1,6 +1,15 @@
 name: Test SQLite versions
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -12,10 +21,10 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest]
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.13"]
         sqlite-version: [
           #"3", # latest version
-          "3.46",
+          #"3.46",
           #"3.45",
           #"3.27",
           #"3.26",

.github/workflows/test.yml

@@ -1,6 +1,15 @@
 name: Test
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -11,16 +20,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
+        include:
+        - python-version: "3.14"
+          coverage: true
     steps:
     - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v6
+      uses: actions/setup-python@v7
       with:
         python-version: ${{ matrix.python-version }}
         allow-prereleases: true
         cache: pip
         cache-dependency-path: pyproject.toml
+        check-latest: true
     - name: Build extension for --load-extension test
       run: |-
         (cd tests && gcc ext.c -fPIC -shared -o ext.so)
@@ -28,12 +41,27 @@ jobs:
       run: |
         pip install . --group dev
         pip freeze
+    - name: Install pytest-cov
+      if: ${{ matrix.coverage }}
+      run: pip install pytest-cov
     - name: Run tests
       run: |
+        if [ "${{ matrix.coverage }}" = "true" ]; then
+          COV="--cov=datasette --cov-config=.coveragerc"
+          pytest -n auto -m "not serial" $COV --cov-report=
+          pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term
+        else
           pytest -n auto -m "not serial"
           pytest -m "serial"
+        fi
         # And the test that exceeds a localhost HTTPS server
         tests/test_datasette_https_server.sh
+    - name: Upload coverage report
+      if: ${{ matrix.coverage }}
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: coverage.xml
     - name: Black
       run: |
         black --version

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.0-slim-bullseye as build
+FROM python:3.11-slim-bookworm AS build
 
 # Version of Datasette to install, e.g. 0.55
 #   docker build . -t datasette --build-arg VERSION=0.55

README.md

@@ -36,7 +36,7 @@ You can also install it using `pip` or `pipx`:
 
     pip install datasette
 
-Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
+Datasette requires Python 3.10 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
 
 ## Basic usage
 

datasette/__init__.py

@@ -1,6 +1,7 @@
 from datasette.permissions import Permission  # noqa
 from datasette.version import __version_info__, __version__  # noqa
 from datasette.events import Event  # noqa
+from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor  # noqa
 from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions  # noqa
 from datasette.utils.asgi import (  # noqa
     Forbidden,

datasette/app.py

@@ -28,7 +28,7 @@ import urllib.parse
 from concurrent import futures
 from pathlib import Path
 
-import httpx
+import httpx2
 from itsdangerous import BadSignature, URLSafeSerializer
 from jinja2 import (
     ChoiceLoader,
@@ -42,6 +42,7 @@ from jinja2.exceptions import TemplateNotFound
 from markupsafe import Markup, escape
 
 from . import stored_queries, write_sql
+from .background_tasks import BackgroundTask, BackgroundTaskSupervisor
 from .column_types import SQLiteType
 from .csrf import CrossOriginProtectionMiddleware
 from .database import Database, QueryInterrupted
@@ -145,6 +146,7 @@ from .views.stored_queries import (
 )
 from .views.table import (
     TableAutocompleteView,
+    TableCountView,
     TableDropView,
     TableFragmentView,
     TableInsertView,
@@ -315,7 +317,7 @@ def _permission_cache_key(actor, action, parent, child):
     actor_key = (
         json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None
     )
-    return (actor_key, action, parent, child)
+    return (actor_key, action.name, parent, action.normalize_child(child))
 
 
 async def favicon(request, send):
@@ -422,6 +424,7 @@ class Datasette:
         default_deny=False,
     ):
         self._startup_invoked = False
+        self._shutdown_invoked = False
         self._closed = False
         assert config_dir is None or isinstance(
             config_dir, Path
@@ -453,8 +456,11 @@ class Datasette:
         self.databases = collections.OrderedDict()
         self.actions = {}  # .invoke_startup() will populate this
         self._column_types = {}  # .invoke_startup() will populate this
+        self._setup_db_done = False
+        self._suppress_background_tasks = False
         try:
             self._refresh_schemas_lock = asyncio.Lock()
+            self._startup_lock = asyncio.Lock()
         except RuntimeError as rex:
             # Workaround for intermittent test failure, see:
             # https://github.com/simonw/datasette/issues/1802
@@ -462,8 +468,10 @@ class Datasette:
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
                 self._refresh_schemas_lock = asyncio.Lock()
+                self._startup_lock = asyncio.Lock()
             else:
                 raise
+        self._background_tasks = BackgroundTaskSupervisor(self)
         self.crossdb = crossdb
         self.nolock = nolock
         if memory or crossdb or not self.files:
@@ -1529,15 +1537,28 @@ class Datasette:
         conn.row_factory = sqlite3.Row
         conn.text_factory = lambda x: str(x, "utf-8", "replace")
         if self.sqlite_extensions and database != INTERNAL_DB_NAME:
+            # Extension loading is only enabled for as long as it takes to
+            # load the configured extensions. Leaving it enabled would let
+            # anyone who can execute SQL call load_extension() themselves.
             conn.enable_load_extension(True)
+            try:
                 for extension in self.sqlite_extensions:
                     # "extension" is either a string path to the extension
                     # or a 2-item tuple that specifies which entrypoint to load.
                     if isinstance(extension, tuple):
                         path, entrypoint = extension
-                    conn.execute("SELECT load_extension(?, ?)", [path, entrypoint])
+                        if sys.version_info >= (3, 12):
+                            conn.load_extension(path, entrypoint=entrypoint)
                         else:
-                    conn.execute("SELECT load_extension(?)", [extension])
+                            # Connection.load_extension() only gained the
+                            # entrypoint argument in Python 3.12
+                            conn.execute(
+                                "SELECT load_extension(?, ?)", [path, entrypoint]
+                            )
+                    else:
+                        conn.load_extension(extension)
+            finally:
+                conn.enable_load_extension(False)
         if self.setting("cache_size_kb"):
             conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}")
         # pylint: disable=no-member
@@ -1730,8 +1751,145 @@ class Datasette:
         sql, params = await build_allowed_resources_sql(
             self, actor, action, parent=parent, include_is_private=include_is_private
         )
+        if action == "view-table":
+            sql, params = await self._apply_derived_table_permissions_to_sql(
+                sql,
+                params,
+                actor=actor,
+                parent=parent,
+                include_is_private=include_is_private,
+            )
         return ResourcesSQL(sql, params)
 
+    async def _allowed_derived_table_source(
+        self, database, source, *, actor, dependencies
+    ):
+        """Check an immediate source, denying sources that are themselves derived."""
+        if any(
+            TableResource.normalize_child(table)
+            == TableResource.normalize_child(source)
+            for table in dependencies
+        ):
+            return False
+        # The source has no dependency in this map. Evaluate its own permission
+        # and prerequisites without starting another dependency check.
+        verdicts = await self._allowed_many(
+            actions=["view-table"],
+            resource=TableResource(database, source),
+            actor=actor,
+            check_derived=False,
+        )
+        return verdicts["view-table"]
+
+    async def _apply_derived_table_permissions_to_sql(
+        self,
+        sql,
+        params,
+        *,
+        actor,
+        parent,
+        include_is_private,
+    ):
+        databases = (
+            [(parent, self.databases[parent])]
+            if parent in self.databases
+            else ([] if parent is not None else list(self.databases.items()))
+        )
+        dependency_maps = dict(
+            zip(
+                (name for name, _ in databases),
+                await asyncio.gather(
+                    *(db.derived_table_dependencies() for _, db in databases)
+                ),
+            )
+        )
+        dependencies = [
+            (database_name, child, source)
+            for database_name, dependency_map in dependency_maps.items()
+            for child, source in dependency_map.items()
+        ]
+        if not dependencies:
+            return sql, params
+
+        sources = sorted(
+            {(database_name, source) for database_name, _, source in dependencies}
+        )
+        actor_verdicts = await asyncio.gather(
+            *(
+                self._allowed_derived_table_source(
+                    database_name,
+                    source,
+                    actor=actor,
+                    dependencies=dependency_maps[database_name],
+                )
+                for database_name, source in sources
+            )
+        )
+        actor_allowed = dict(zip(sources, actor_verdicts))
+
+        anonymous_allowed = {}
+        if include_is_private:
+            anonymous_verdicts = await asyncio.gather(
+                *(
+                    self._allowed_derived_table_source(
+                        database_name,
+                        source,
+                        actor=None,
+                        dependencies=dependency_maps[database_name],
+                    )
+                    for database_name, source in sources
+                )
+            )
+            anonymous_allowed = dict(zip(sources, anonymous_verdicts))
+
+        wrapped_params = dict(params)
+        derived_rows = [
+            [
+                database_name,
+                child,
+                int(actor_allowed[(database_name, source)]),
+                *(
+                    [int(anonymous_allowed[(database_name, source)])]
+                    if include_is_private
+                    else []
+                ),
+            ]
+            for database_name, child, source in dependencies
+        ]
+        derived_param = "_datasette_derived_permissions"
+        while derived_param in wrapped_params:
+            derived_param += "_"
+        wrapped_params[derived_param] = json.dumps(derived_rows)
+
+        derived_columns = "parent, child, source_allowed"
+        select_columns = "allowed.parent, allowed.child, allowed.reason"
+        if include_is_private:
+            derived_columns += ", source_anonymous_allowed"
+            select_columns += (
+                ", CASE WHEN derived.source_anonymous_allowed = 0 "
+                "THEN 1 ELSE allowed.is_private END AS is_private"
+            )
+        wrapped_sql = f"""
+WITH derived_permissions({derived_columns}) AS (
+  SELECT
+    json_extract(value, '$[0]'),
+    json_extract(value, '$[1]'),
+    json_extract(value, '$[2]')
+    {", json_extract(value, '$[3]')" if include_is_private else ""}
+  FROM json_each(:{derived_param})
+),
+allowed AS (
+{sql}
+)
+SELECT {select_columns}
+FROM allowed
+LEFT JOIN derived_permissions AS derived
+  ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE
+WHERE COALESCE(derived.source_allowed, 1) = 1
+ORDER BY allowed.parent, allowed.child
+""".strip()
+        return wrapped_sql, wrapped_params
+
     async def allowed_resources(
         self,
         action: str,
@@ -1934,6 +2092,12 @@ class Datasette:
             )
             # {"edit-schema": True, "drop-table": True, "insert-row": False}
         """
+        return await self._allowed_many(
+            actions=actions, resource=resource, actor=actor, check_derived=True
+        )
+
+    async def _allowed_many(self, *, actions, resource, actor, check_derived):
+        """Evaluate permissions, optionally applying the one-hop source policy."""
         from datasette.permissions import (
             _permission_check_cache,
             _skip_permission_checks,
@@ -1971,7 +2135,7 @@ class Datasette:
         to_check = []
         for name in expanded:
             if cache is not None:
-                key = _permission_cache_key(actor, name, parent, child)
+                key = _permission_cache_key(actor, self.actions[name], parent, child)
                 if key in cache:
                     final[name] = cache[key]
                     continue
@@ -1987,6 +2151,28 @@ class Datasette:
                 child=child,
             )
 
+        if (
+            check_derived
+            and "view-table" in to_check
+            and raw.get("view-table")
+            and isinstance(resource, TableResource)
+            and parent in self.databases
+        ):
+            dependencies = await self.databases[parent].derived_table_dependencies()
+            source = next(
+                (
+                    source
+                    for table, source in dependencies.items()
+                    if TableResource.normalize_child(table)
+                    == TableResource.normalize_child(child)
+                ),
+                None,
+            )
+            if source is not None:
+                raw["view-table"] = await self._allowed_derived_table_source(
+                    parent, source, actor=actor, dependencies=dependencies
+                )
+
         def resolve(name):
             # final verdict = own rules AND verdict of also_requires chain
             if name in final:
@@ -2004,7 +2190,9 @@ class Datasette:
         # Cache the freshly computed checks
         if cache is not None:
             for name in to_check:
-                cache[_permission_cache_key(actor, name, parent, child)] = final[name]
+                cache[
+                    _permission_cache_key(actor, self.actions[name], parent, child)
+                ] = final[name]
 
         # Log every check (including cache hits) for the debug page,
         # dependencies before the actions that required them
@@ -2278,6 +2466,21 @@ class Datasette:
         )
         return d
 
+    def _tasks(self):
+        return {
+            "tasks": [
+                {
+                    "name": t.name,
+                    "state": t.state,
+                    "function": t.function,
+                    "started_at": t.started_at,
+                    "exception": repr(t.exception) if t.exception else None,
+                }
+                for t in self._background_tasks.tasks()
+            ],
+            "launched": self._background_tasks.launched,
+        }
+
     def _actor(self, request):
         return {"actor": request.actor}
 
@@ -2384,6 +2587,8 @@ class Datasette:
             datasette=self,
         ):
             extra_vars = await await_me_maybe(extra_vars)
+            if extra_vars is None:
+                continue
             assert isinstance(
                 extra_vars, dict
             ), f"extra_vars is of type {type(extra_vars)}"
@@ -2446,7 +2651,7 @@ class Datasette:
     ):
         data = {"a": actor}
         if expire_after:
-            expires_at = int(time.time()) + (24 * 60 * 60)
+            expires_at = int(time.time()) + expire_after
             data["e"] = baseconv.base62.encode(expires_at)
         response.set_cookie("ds_actor", self.sign(data, "actor"))
 
@@ -2566,6 +2771,12 @@ class Datasette:
             ),
             r"/-/threads(\.(?P<format>json))?$",
         )
+        add_route(
+            JsonDataView.as_view(
+                self, "tasks.json", self._tasks, permission="permissions-debug"
+            ),
+            r"/-/tasks(\.(?P<format>json))?$",
+        )
         add_route(
             JsonDataView.as_view(
                 self,
@@ -2740,6 +2951,10 @@ class Datasette:
             TableSetColumnTypeView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/set-column-type$",
         )
+        add_route(
+            TableCountView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/count$",
+        )
         add_route(
             TableFragmentView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/fragment$",
@@ -2803,26 +3018,127 @@ class Datasette:
             raise RowNotFound(db.name, table_name, pk_values)
         return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first())
 
+    async def _startup_sequence(self):
+        """Idempotently run the full startup sequence: table counts for
+        immutable databases, then invoke_startup(). Safe to call more than
+        once and safe to call concurrently - callers block until whichever
+        call got there first has finished.
+
+        This is the single entry point used by both AsgiLifespan (so
+        real deployments finish startup before accepting requests) and
+        AsgiRunOnFirstRequest (the fallback for hosts that never send
+        lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and
+        `datasette serve` (cli.py) calls it too. The fast path below checks
+        both `_startup_invoked` and `_setup_db_done` - not just the former -
+        so that a bare `await ds.invoke_startup()` made by a caller ahead of
+        `_startup_sequence()` (which only sets `_startup_invoked`) can't
+        make this method skip the immutable-database table-count precompute.
+        """
+        if self._startup_invoked and self._setup_db_done:
+            return
+        async with self._startup_lock:
+            if self._startup_invoked and self._setup_db_done:
+                return
+            if not self._setup_db_done:
+                # First time server starts up, calculate table counts for
+                # immutable databases
+                for database in self.databases.values():
+                    if not database.is_mutable:
+                        await database.table_counts(limit=60 * 60 * 1000)
+                self._setup_db_done = True
+            await self.invoke_startup()
+
+    def add_background_task(self, func, name=None) -> BackgroundTask:
+        """Register a piece of supervised background work, typically from
+        a plugin's ``startup`` hook.
+
+        ``func`` must be a coroutine function taking one positional
+        argument, the ``Datasette`` instance - core calls ``func(self)``.
+        Callable any time after ``__init__``: if background tasks haven't
+        launched yet (the common case - most callers are ``startup`` hooks,
+        which run before launch), this buffers the registration until they
+        do; if they've already launched (e.g. called from a request
+        handler after the server is up), the task starts immediately.
+
+        Returns a :class:`~datasette.background_tasks.BackgroundTask`
+        handle (``.name``, ``.state``, ``.task``, ``.exception``,
+        ``.started_at``, ``.function``, ``.cancel()``).
+
+        ``name`` defaults to ``func.__qualname__``; on a name collision a
+        ``-2``, ``-3``, ... suffix is appended, since names are how
+        ``/-/tasks`` and log messages identify work.
+        """
+        return self._background_tasks.add(func, name=name)
+
+    async def start_background_tasks(self):
+        """Run startup (if it hasn't run yet) and launch every registered
+        background task.
+
+        Public entry point for tests, embedders, and headless CLIs (the
+        ``datasette-rss``-style ``fetch --due`` shape) that want supervised
+        background tasks without running a server - equivalent to what
+        happens automatically via ASGI lifespan / the first-request
+        fallback in a served deployment.
+        """
+        await self.invoke_startup()
+        await self._background_tasks.launch_all()
+
+    async def _launch_background_tasks(self):
+        """Idempotently launch every registered background task. Private:
+        this is the entry point wired into the lifecycle trigger lists
+        (the second entry in both ``AsgiLifespan`` and
+        ``AsgiRunOnFirstRequest``'s ``on_startup``, after
+        ``_startup_sequence``) - not something plugins or embedders should
+        call directly; use ``add_background_task`` /
+        ``start_background_tasks`` instead.
+
+        Positioned after ``_startup_sequence`` in both trigger lists so
+        launch always happens once every plugin's ``startup`` hook has had
+        a chance to register work - the ordering guarantee that makes
+        ``add_background_task`` useful. No-ops when
+        ``_suppress_background_tasks`` is set (the ``--get`` CLI path: its
+        one-shot TestClient request flows through the full ASGI stack,
+        including the first-request fallback, but must never launch
+        long-lived background work).
+        """
+        if self._suppress_background_tasks:
+            return
+        await self._background_tasks.launch_all()
+
+    async def invoke_shutdown(self):
+        """Run the graceful teardown sequence: plugin ``shutdown`` hooks,
+        then cancel and drain supervised background tasks, then close
+        every database.
+        """
+        if self._shutdown_invoked:
+            return
+        self._shutdown_invoked = True
+        for hook in pm.hook.shutdown(datasette=self):
+            try:
+                await await_me_maybe(hook)
+            except Exception:
+                logging.getLogger("datasette").exception("shutdown hook failed")
+        await self._background_tasks.cancel_all(grace=5.0)
+        self.close()
+
     def app(self):
         """Returns an ASGI app function that serves the whole of Datasette"""
         routes = self._routes()
 
-        async def setup_db():
-            # First time server starts up, calculate table counts for immutable databases
-            for database in self.databases.values():
-                if not database.is_mutable:
-                    await database.table_counts(limit=60 * 60 * 1000)
-
-        async def _close_on_shutdown():
-            self.close()
-
         asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self)
         if self.setting("trace_debug"):
             asgi = AsgiTracer(asgi)
-        asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown])
-        asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup])
+        asgi = AsgiLifespan(
+            asgi,
+            on_startup=[self._startup_sequence, self._launch_background_tasks],
+            on_shutdown=[self.invoke_shutdown],
+        )
         for wrapper in pm.hook.asgi_wrapper(datasette=self):
             asgi = wrapper(asgi)
+        asgi = AsgiRunOnFirstRequest(
+            asgi,
+            on_startup=[self._startup_sequence, self._launch_background_tasks],
+        )
         return asgi
 
 
@@ -2860,6 +3176,50 @@ class DatasetteRouter:
             receive,
             max_post_body_bytes=self.ds.setting("max_post_body_bytes"),
         )
+        match, view = resolve_routes(self.routes, path)
+        is_static = view is favicon or getattr(view, "_datasette_static", False)
+        original_send = send
+
+        async def send(message):
+            if message["type"] == "http.response.start" and not (
+                is_static and message["status"] in (200, 304)
+            ):
+                # Decide privacy after rendering, including for streaming responses
+                # and error handlers. A public primary resource can still include
+                # private labels, actor navigation, or cookie-dependent content.
+                headers = list(message.get("headers", []))
+                personalized = (
+                    request.actor is not None
+                    or "cookie" in request.headers
+                    or "authorization" in request.headers
+                    or any(key.lower() == b"set-cookie" for key, _ in headers)
+                )
+                if personalized:
+                    headers = [
+                        (key, value)
+                        for key, value in headers
+                        if key.lower() != b"cache-control"
+                    ]
+                    headers.append((b"cache-control", b"private, no-store"))
+
+                # Anonymous responses must not be reused for credentialed requests.
+                # Preserve any additional variation specified by views or plugins.
+                vary = [
+                    part.strip()
+                    for key, value in headers
+                    if key.lower() == b"vary"
+                    for part in value.split(b",")
+                    if part.strip()
+                ]
+                if b"*" not in vary:
+                    for name in (b"Cookie", b"Authorization"):
+                        if name.lower() not in {part.lower() for part in vary}:
+                            vary.append(name)
+                headers = [(k, v) for k, v in headers if k.lower() != b"vary"]
+                headers.append((b"vary", b", ".join(vary)))
+                message = dict(message, headers=headers)
+            await original_send(message)
+
         # Populate request_messages if ds_messages cookie is present
         try:
             request._messages = self.ds.unsign(
@@ -2899,8 +3259,7 @@ class DatasetteRouter:
             return await self.handle_401(request, send, token_error)
         scope_modifications["actor"] = actor or default_actor
         scope = dict(scope, **scope_modifications)
-
-        match, view = resolve_routes(self.routes, path)
+        request.scope = scope
 
         if match is None:
             return await self.handle_404(request, send)
@@ -3215,14 +3574,14 @@ class DatasetteClient:
         with _DatasetteClientContext():
             if skip_permission_checks:
                 with SkipPermissions():
-                    async with httpx.AsyncClient(
-                        transport=httpx.ASGITransport(app=self.app),
+                    async with httpx2.AsyncClient(
+                        transport=httpx2.ASGITransport(app=self.app),
                         cookies=kwargs.pop("cookies", None),
                     ) as client:
                         return await getattr(client, method)(self._fix(path), **kwargs)
             else:
-                async with httpx.AsyncClient(
-                    transport=httpx.ASGITransport(app=self.app),
+                async with httpx2.AsyncClient(
+                    transport=httpx2.ASGITransport(app=self.app),
                     cookies=kwargs.pop("cookies", None),
                 ) as client:
                     return await getattr(client, method)(self._fix(path), **kwargs)
@@ -3269,10 +3628,10 @@ class DatasetteClient:
             method: HTTP method (e.g., "GET", "POST", "PUT")
             path: The path to request
             skip_permission_checks: If True, bypass all permission checks for this request
-            **kwargs: Additional arguments to pass to httpx
+            **kwargs: Additional arguments to pass to httpx2
 
         Returns:
-            httpx.Response: The response from the request
+            httpx2.Response: The response from the request
         """
         from datasette.permissions import SkipPermissions
 
@@ -3281,16 +3640,16 @@ class DatasetteClient:
         with _DatasetteClientContext():
             if skip_permission_checks:
                 with SkipPermissions():
-                    async with httpx.AsyncClient(
-                        transport=httpx.ASGITransport(app=self.app),
+                    async with httpx2.AsyncClient(
+                        transport=httpx2.ASGITransport(app=self.app),
                         cookies=kwargs.pop("cookies", None),
                     ) as client:
                         return await client.request(
                             method, self._fix(path, avoid_path_rewrites), **kwargs
                         )
             else:
-                async with httpx.AsyncClient(
-                    transport=httpx.ASGITransport(app=self.app),
+                async with httpx2.AsyncClient(
+                    transport=httpx2.ASGITransport(app=self.app),
                     cookies=kwargs.pop("cookies", None),
                 ) as client:
                     return await client.request(

datasette/background_tasks.py

@@ -0,0 +1,227 @@
+"""
+Supervised background-task registration for Datasette core.
+
+Plugins that need long-lived background work (a polling loop, a queue
+consumer, a scheduled job runner) register it with
+``datasette.add_background_task(func, name=None)`` - typically from a
+``startup`` plugin hook - instead of fire-and-forgetting their own
+``asyncio.create_task()``. Core owns:
+
+- **references**: every launched ``asyncio.Task`` is kept alive on a
+  :class:`BackgroundTaskSupervisor`, so it can never be silently garbage
+  collected the way an unreferenced ``create_task()`` call can be;
+- **launch timing**: registered work is buffered until
+  :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to
+  happen only after *every* plugin's ``startup`` hook has finished - so
+  a task that depends on another plugin having registered something first
+  doesn't need ``tryfirst=True`` ordering tricks;
+- **crash surfacing**: an unhandled exception in a background task is
+  logged with its full traceback to the ``datasette.background_tasks``
+  logger and recorded on the handle, instead of becoming an "Task
+  exception was never retrieved" warning nobody sees;
+- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels
+  every task still running and waits (with a grace period) for them to
+  actually stop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import datetime
+import functools
+import logging
+from collections.abc import Awaitable, Callable
+
+logger = logging.getLogger("datasette.background_tasks")
+
+
+def _utcnow_iso() -> str:
+    return datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+
+def _function_path(func: Callable) -> str:
+    """Describe the callable without guessing which plugin registered it."""
+    while isinstance(func, functools.partial):
+        func = func.func
+    if not hasattr(func, "__qualname__"):
+        func = type(func).__call__
+    return f"{func.__module__}.{func.__qualname__}"
+
+
+class BackgroundTask:
+    """A handle to a single piece of supervised background work.
+
+    States: ``registered`` (added but not yet launched) -> ``running`` ->
+    one of ``completed`` (returned cleanly), ``crashed`` (raised an
+    exception other than ``CancelledError`` - see ``.exception``), or
+    ``cancelled`` (``.cancel()`` was called, or it was still running at
+    shutdown).
+    """
+
+    def __init__(
+        self,
+        name: str,
+        func: Callable[[object], Awaitable[None]],
+    ):
+        self.name = name
+        self.state = "registered"
+        self.task: asyncio.Task | None = None
+        self.exception: BaseException | None = None
+        self.started_at: str | None = None
+        self.function = _function_path(func)
+        self._func = func
+        self._supervisor: BackgroundTaskSupervisor | None = None
+
+    def cancel(self) -> None:
+        """Cancel this task.
+
+        If it has already been launched, cancels the underlying
+        ``asyncio.Task`` - its state becomes ``cancelled`` once the
+        cancellation is observed (asynchronously, via the task's done
+        callback). If it has not been launched yet, this is a no-op as
+        far as asyncio is concerned (there's no task to cancel) but it
+        deregisters the handle from its supervisor so it never runs.
+        """
+        if self.task is not None:
+            self.task.cancel()
+        elif self._supervisor is not None:
+            self._supervisor._deregister(self)
+
+    def __repr__(self) -> str:
+        return f"<BackgroundTask name={self.name!r} state={self.state!r}>"
+
+
+class BackgroundTaskSupervisor:
+    """Owns registration and launch of every :class:`BackgroundTask` for a
+    single ``Datasette`` instance.
+
+    Registration (:meth:`add`) is separate from launch
+    (:meth:`launch_all`): plugins register work whenever convenient
+    (typically from a ``startup`` hook, but request handlers can register
+    dynamic per-job work too), and it either sits buffered until
+    :meth:`launch_all` runs, or - if :meth:`launch_all` has already run -
+    starts immediately.
+
+    Strong references to every :class:`BackgroundTask` (and its
+    ``asyncio.Task``) are kept for the life of the instance, by design -
+    that's what makes the enrichments-style "fire-and-forget task gets
+    garbage collected mid-flight" bug impossible here. There is currently
+    no pruning of completed/crashed/cancelled tasks, so a plugin that
+    dynamically registers many short-lived tasks over a long process
+    lifetime (a per-job registration pattern, e.g. one task per queued
+    job) will grow this list without bound. That's an accepted v1
+    trade-off in favour of full introspection (``/-/tasks``); revisit
+    with a pruning or capping policy if unbounded growth is reported in
+    practice.
+    """
+
+    def __init__(self, datasette):
+        self._datasette = datasette
+        self._tasks: list[BackgroundTask] = []
+        self._names = set()
+        self._launched = False
+        self._lock = asyncio.Lock()
+
+    def add(self, func, name=None) -> BackgroundTask:
+        base_name = name or getattr(func, "__qualname__", None) or repr(func)
+        actual_name = self._unique_name(base_name)
+        handle = BackgroundTask(actual_name, func)
+        handle._supervisor = self
+        self._tasks.append(handle)
+        self._names.add(actual_name)
+        if self._launched:
+            self._launch_one(handle)
+        return handle
+
+    def _unique_name(self, base_name: str) -> str:
+        if base_name not in self._names:
+            return base_name
+        n = 2
+        while f"{base_name}-{n}" in self._names:
+            n += 1
+        return f"{base_name}-{n}"
+
+    def _deregister(self, handle: BackgroundTask) -> None:
+        try:
+            self._tasks.remove(handle)
+        except ValueError:
+            pass
+        self._names.discard(handle.name)
+
+    def _launch_one(self, handle: BackgroundTask) -> None:
+        handle.state = "running"
+        handle.started_at = _utcnow_iso()
+        handle.task = asyncio.create_task(
+            handle._func(self._datasette), name=handle.name
+        )
+        handle.task.add_done_callback(functools.partial(_on_task_done, handle))
+
+    async def launch_all(self) -> None:
+        """Launch every currently-registered task that hasn't launched
+        yet. Idempotent and safe to call concurrently: subsequent (or
+        racing) calls are no-ops once the first has set ``self._launched``.
+        """
+        if self._launched:
+            return
+        async with self._lock:
+            if self._launched:
+                return
+            self._launched = True
+            for handle in list(self._tasks):
+                if handle.task is None:
+                    self._launch_one(handle)
+
+    async def cancel_all(self, grace: float = 5.0) -> None:
+        """Cancel every task that isn't already done, then wait up to
+        ``grace`` seconds for them to actually finish. Stragglers still
+        running after that are logged by name (but left to finish or not
+        on their own - this does not forcibly kill them, asyncio has no
+        mechanism for that).
+        """
+        handles_by_task = {
+            handle.task: handle for handle in self._tasks if handle.task is not None
+        }
+        pending = [task for task in handles_by_task if not task.done()]
+        for task in pending:
+            task.cancel()
+        if not pending:
+            return
+        _done, not_done = await asyncio.wait(pending, timeout=grace)
+        if not_done:
+            names = sorted(handles_by_task[task].name for task in not_done)
+            logger.warning(
+                "%d background task(s) did not finish within the %.1fs grace "
+                "period after cancellation: %s",
+                len(names),
+                grace,
+                ", ".join(names),
+            )
+
+    def tasks(self) -> list[BackgroundTask]:
+        """Return every registered :class:`BackgroundTask`, launched or
+        not, in registration order. Used by the ``/-/tasks`` debug
+        endpoint.
+        """
+        return list(self._tasks)
+
+    @property
+    def launched(self) -> bool:
+        """Whether :meth:`launch_all` has run yet - lets ``/-/tasks``
+        distinguish "no tasks registered" from "tasks registered but
+        nothing has armed the launch yet" without reaching for the
+        private ``_launched`` attribute.
+        """
+        return self._launched
+
+
+def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None:
+    if task.cancelled():
+        handle.state = "cancelled"
+        return
+    exc = task.exception()
+    if exc is not None:
+        handle.state = "crashed"
+        handle.exception = exc
+        logger.error("Background task %r crashed", handle.name, exc_info=exc)
+        return
+    handle.state = "completed"

datasette/cli.py

@@ -497,6 +497,7 @@ def uninstall(packages, yes):
     "--internal",
     type=click.Path(),
     help="Path to a persistent Datasette internal SQLite database",
+    envvar="DATASETTE_INTERNAL",
 )
 def serve(
     files,
@@ -663,16 +664,6 @@ def serve(
         # Private utility mechanism for writing unit tests
         return ds
 
-    # Run async soundness checks before startup hooks, since invoke_startup
-    # now populates internal tables which requires querying each database
-    run_sync(lambda: check_databases(ds))
-
-    # Run the "startup" plugin hooks
-    try:
-        run_sync(ds.invoke_startup)
-    except StartupError as e:
-        raise click.ClickException(e.args[0])
-
     if headers and not get:
         raise click.ClickException("--headers can only be used with --get")
 
@@ -680,6 +671,19 @@ def serve(
         raise click.ClickException("--token can only be used with --get")
 
     if get:
+        # --get means we don't run Uvicorn at all
+        run_sync(lambda: check_databases(ds))
+
+        try:
+            run_sync(ds.invoke_startup)
+        except StartupError as e:
+            raise click.ClickException(e.args[0])
+
+        # --get never launches background tasks: TestClient's request below
+        # flows through the full ASGI stack, including the
+        # AsgiRunOnFirstRequest fallback, which would otherwise launch them.
+        ds._suppress_background_tasks = True
+
         client = TestClient(ds)
         request_headers = {}
         if token:
@@ -704,6 +708,23 @@ def serve(
         sys.exit(exit_code)
         return
 
+    # check_databases, invoke_startup() and the uvicorn server all run on a
+    # single event loop, so that anything a plugin's "startup" hook schedules
+    # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is
+    # still alive when the server starts handling requests.
+    async def _serve_async():
+        # Populate internal catalog tables before invoke_startup
+        await check_databases(ds)
+
+        # Run the full startup sequence (immutable-database table-count
+        # precompute + the "startup" plugin hooks) via the same entry point
+        # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when
+        # uvicorn's lifespan.startup fires moments later.
+        try:
+            await ds._startup_sequence()
+        except StartupError as e:
+            raise click.ClickException(e.args[0])
+
         # Start the server
         url = None
         if root:
@@ -715,7 +736,7 @@ def serve(
         if open_browser:
             if url is None:
                 # Figure out most convenient URL - to table, database or homepage
-            path = run_sync(lambda: initial_path_for_datasette(ds))
+                path = await initial_path_for_datasette(ds)
                 url = f"http://{host}:{port}{path}"
             webbrowser.open(url)
         uvicorn_kwargs = {
@@ -731,7 +752,10 @@ def serve(
             uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile
         if ssl_certfile:
             uvicorn_kwargs["ssl_certfile"] = ssl_certfile
-    uvicorn.run(ds.app(), **uvicorn_kwargs)
+        server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs))
+        await server.serve()
+
+    asyncio.run(_serve_async())
 
 
 @cli.command()

datasette/database.py

@@ -29,7 +29,7 @@ from .utils import (
     table_columns,
 )
 from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables
-from .utils.sqlite import sqlite_hidden_table_names
+from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names
 
 connections = threading.local()
 
@@ -85,6 +85,7 @@ class Database:
         self.cached_hash = None
         self.cached_size = None
         self._cached_table_counts = None
+        self._cached_derived_table_dependencies = None
         self._write_thread = None
         self._write_queue = None
         self._closed = False
@@ -93,8 +94,9 @@ class Database:
         # These are used when in non-threaded mode:
         self._read_connection = None
         self._write_connection = None
-        # This is used to track all file connections so they can be closed
-        self._all_file_connections = []
+        # Track file and memory connections, including reads on worker threads,
+        # so close() can release all of them from the calling thread.
+        self._all_connections = []
         if not is_temp_disk:
             self.mode = mode
 
@@ -145,9 +147,12 @@ class Database:
             )
             if not write:
                 conn.execute("PRAGMA query_only=1")
+            self._all_connections.append(conn)
             return conn
         if self.is_memory:
-            return sqlite3.connect(":memory:", uri=True)
+            conn = sqlite3.connect(":memory:", uri=True, check_same_thread=False)
+            self._all_connections.append(conn)
+            return conn
 
         # mode=ro or immutable=1?
         if self.is_mutable:
@@ -164,7 +169,7 @@ class Database:
         conn = sqlite3.connect(
             f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs
         )
-        self._all_file_connections.append(conn)
+        self._all_connections.append(conn)
         if self.is_temp_disk and not self._wal_enabled:
             conn.execute("PRAGMA journal_mode=WAL")
             self._wal_enabled = True
@@ -201,13 +206,13 @@ class Database:
             except Exception:  # noqa: BLE001, S110
                 # Shutdown teardown - a failed pending write must not block close()
                 pass
-        # Close anything still tracked in _all_file_connections
-        for connection in self._all_file_connections:
+        # Close anything still tracked in _all_connections
+        for connection in self._all_connections:
             try:
                 connection.close()
             except Exception:  # noqa: BLE001, S110
                 pass
-        self._all_file_connections = []
+        self._all_connections = []
         # Drop per-thread cached read connections we can reach
         try:
             delattr(connections, self._thread_local_id)
@@ -246,17 +251,29 @@ class Database:
         return_all=False,
         returning_limit=EXECUTE_WRITE_RETURNING_LIMIT,
         transaction=True,
+        time_limit_ms=2000,
     ):
         self._check_not_closed()
         if returning_limit < 0:
             raise ValueError("returning_limit must be >= 0")
 
-        def _inner(conn):
+        def execute_sql(conn):
             cursor = conn.execute(sql, params or [])
             return ExecuteWriteResult.from_cursor(
                 cursor, return_all=return_all, returning_limit=returning_limit
             )
 
+        def _inner(conn):
+            try:
+                if time_limit_ms is None:
+                    return execute_sql(conn)
+                with sqlite_timelimit(conn, time_limit_ms):
+                    return execute_sql(conn)
+            except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:
+                if e.args == ("interrupted",):
+                    raise QueryInterrupted(e, sql, params)
+                raise
+
         with trace("sql", database=self.name, sql=sql.strip(), params=params):
             results = await self.execute_write_fn(
                 _inner, block=block, request=request, transaction=transaction
@@ -311,9 +328,9 @@ class Database:
             finally:
                 isolated_connection.close()
                 try:
-                    self._all_file_connections.remove(isolated_connection)
+                    self._all_connections.remove(isolated_connection)
                 except ValueError:
-                    # Was probably a memory connection
+                    # May already have been cleared by close().
                     pass
 
         if self.ds.executor is None:
@@ -354,6 +371,15 @@ class Database:
                     result = fn(self._write_connection)
             else:
                 result = fn(self._write_connection)
+            if not block:
+                # There is no write thread here, so the write has already
+                # finished. Hand back the same (task_id, reply_future) shape
+                # _send_to_write_thread() returns, with the future already
+                # resolved, so the block=False path below is identical in
+                # both modes.
+                reply_future = asyncio.get_running_loop().create_future()
+                reply_future.set_result(result)
+                result = (uuid.uuid4(), reply_future)
         else:
             result = await self._send_to_write_thread(
                 fn, block=block, transaction=transaction
@@ -425,7 +451,7 @@ class Database:
             )
             self._write_thread.name = f"_execute_writes for database {self.name}"
             self._write_thread.start()
-        task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
+        task_id = uuid.uuid4()
         loop = asyncio.get_running_loop()
         reply_future = loop.create_future()
         self._write_queue.put(
@@ -469,9 +495,9 @@ class Database:
                     finally:
                         isolated_connection.close()
                         try:
-                            self._all_file_connections.remove(isolated_connection)
+                            self._all_connections.remove(isolated_connection)
                         except ValueError:
-                            # Was probably a memory connection
+                            # May already have been cleared by close().
                             pass
                 except Exception as e:  # noqa: BLE001
                     # Write thread must survive any task failure or the database wedges
@@ -759,6 +785,17 @@ class Database:
 
         return hidden_tables
 
+    async def derived_table_dependencies(self):
+        """Return implementation tables and the tables they derive from."""
+        schema_version = (await self.execute("PRAGMA schema_version")).first()[0]
+        if (
+            self._cached_derived_table_dependencies is None
+            or self._cached_derived_table_dependencies[0] != schema_version
+        ):
+            dependencies = await self.execute_fn(sqlite_derived_table_dependencies)
+            self._cached_derived_table_dependencies = (schema_version, dependencies)
+        return self._cached_derived_table_dependencies[1]
+
     async def view_names(self):
         results = await self.execute("select name from sqlite_master where type='view'")
         return [r[0] for r in results.rows]

datasette/default_column_types.py

@@ -6,6 +6,17 @@ import markupsafe
 from datasette import hookimpl
 from datasette.column_types import ColumnType, SQLiteType
 
+_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE)
+
+
+def _normalize_http_url(value):
+    if not isinstance(value, str):
+        return None
+    normalized = value.strip()
+    if not _HTTP_URL_RE.fullmatch(normalized):
+        return None
+    return normalized
+
 
 class UrlColumnType(ColumnType):
     name = "url"
@@ -15,7 +26,10 @@ class UrlColumnType(ColumnType):
     async def render_cell(self, value, column, table, database, datasette, request):
         if not value or not isinstance(value, str):
             return None
-        escaped = markupsafe.escape(value.strip())
+        normalized = _normalize_http_url(value)
+        if normalized is None:
+            return markupsafe.escape(value.strip())
+        escaped = markupsafe.escape(normalized)
         return markupsafe.Markup(f'<a href="{escaped}">{escaped}</a>')
 
     async def validate(self, value, datasette):
@@ -23,7 +37,7 @@ class UrlColumnType(ColumnType):
             return None
         if not isinstance(value, str):
             return "URL must be a string"
-        if not re.match(r"^https?://\S+$", value.strip()):
+        if _normalize_http_url(value) is None:
             return "Invalid URL"
         return None
 

datasette/default_permissions/config.py

@@ -92,6 +92,13 @@ class ConfigPermissionProcessor:
             # Tables implicitly reference their parent databases
             self.restricted_databases.update(db for db, _ in self.restricted_tables)
 
+        # Resolve identity keys once per action, rather than scanning the
+        # restriction allowlist for every configured table's allow block.
+        self.restricted_table_keys = {
+            (db, self.action_obj.normalize_child(table) if self.action_obj else table)
+            for db, table in self.restricted_tables
+        }
+
     def evaluate_allow_block(self, allow_block: Any) -> bool | None:
         """Evaluate an allow block against the current actor."""
         if allow_block is None:
@@ -125,8 +132,10 @@ class ConfigPermissionProcessor:
         if parent:
             table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {})
             if child:
-                table_actions = table_restrictions.get(child, [])
-                if self.action_checks.intersection(table_actions):
+                child_key = (
+                    self.action_obj.normalize_child(child) if self.action_obj else child
+                )
+                if (parent, child_key) in self.restricted_table_keys:
                     return True
             else:
                 # Parent query should proceed if any child in this database is allowlisted

datasette/default_permissions/restrictions.py

@@ -185,8 +185,12 @@ def restrictions_allow_action(
     # Check table/resource level
     if resource is not None and not isinstance(resource, str) and len(resource) == 2:
         database, table = resource
-        table_allowed = restrictions.get("r", {}).get(database, {}).get(table)
-        if table_allowed is not None:
+        action_obj = datasette.actions.get(action)
+        normalize = action_obj.normalize_child if action_obj else lambda name: name
+        for table_name, table_allowed in (
+            restrictions.get("r", {}).get(database, {}).items()
+        ):
+            if normalize(table_name) == normalize(table):
                 assert isinstance(table_allowed, list)
                 if to_check.intersection(table_allowed):
                     return True

datasette/default_permissions/sqlite_statistics.py

@@ -0,0 +1,25 @@
+"""Default table-access policy for SQLite optimizer statistics."""
+
+import json
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+
+@hookimpl
+def permission_resources_sql(action):
+    if action != "view-table":
+        return None
+    return PermissionSQL(
+        sql="""
+            SELECT database_name AS parent, value AS child, 0 AS allow,
+                'SQLite statistics tables are denied by default' AS reason
+            FROM catalog_databases
+            CROSS JOIN json_each(:sqlite_statistics_names)
+        """,
+        params={
+            "sqlite_statistics_names": json.dumps(
+                ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"]
+            )
+        },
+    )

datasette/facets.py

@@ -264,10 +264,15 @@ class ColumnFacet(Facet):
                     column_qs = column
                     if column.startswith("_"):
                         column_qs = f"{column}__exact"
-                    selected = (column_qs, str(row["value"])) in qs_pairs
+                    selected_args = {
+                        key: str(row["value"])
+                        for key in (column_qs, f"{column}__exact")
+                        if (key, str(row["value"])) in qs_pairs
+                    }
+                    selected = bool(selected_args)
                     if selected:
                         toggle_path = path_with_removed_args(
-                            self.request, {column_qs: str(row["value"])}
+                            self.request, selected_args
                         )
                     else:
                         toggle_path = path_with_added_args(

datasette/filters.py

@@ -1,8 +1,9 @@
 import json
+import math
 from typing import ClassVar
 
 from datasette import hookimpl
-from datasette.resources import DatabaseResource
+from datasette.resources import DatabaseResource, TableResource
 from datasette.utils.asgi import BadRequest
 from datasette.views.base import DatasetteError
 
@@ -51,13 +52,20 @@ def search_filters(request, database, table, datasette):
         human_descriptions = []
         extra_context = {}
 
-        # Figure out which fts_table to use
+        # Figure out which trusted fts_table to use. Query string parameters can
+        # repeat this mapping (for backwards compatibility), but must not select
+        # a different table or primary key.
         table_metadata = await datasette.table_config(database, table)
         db = datasette.get_database(database)
-        fts_table = request.args.get("_fts_table")
-        fts_table = fts_table or table_metadata.get("fts_table")
+        fts_table = table_metadata.get("fts_table")
         fts_table = fts_table or await db.fts_table(table)
-        fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid"))
+        fts_pk = table_metadata.get("fts_pk", "rowid")
+        requested_fts_table = request.args.get("_fts_table")
+        requested_fts_pk = request.args.get("_fts_pk")
+        if (requested_fts_table and requested_fts_table != fts_table) or (
+            requested_fts_pk and requested_fts_pk != fts_pk
+        ):
+            raise BadRequest("Invalid _fts_table or _fts_pk")
         search_args = {
             key: request.args[key]
             for key in request.args
@@ -75,6 +83,11 @@ def search_filters(request, database, table, datasette):
         extra_context["supports_search"] = bool(fts_table)
 
         if fts_table and search_args:
+            await datasette.ensure_permission(
+                action="view-table",
+                resource=TableResource(database=database, table=fts_table),
+                actor=request.actor,
+            )
             if "_search" in search_args:
                 # Simple ?_search=xxx
                 search = search_args["_search"]
@@ -135,6 +148,11 @@ def through_filters(request, database, table, datasette):
                 through_table = through_data["table"]
                 other_column = through_data["column"]
                 value = through_data["value"]
+                await datasette.ensure_permission(
+                    action="view-table",
+                    resource=TableResource(database=database, table=through_table),
+                    actor=request.actor,
+                )
                 db = datasette.get_database(database)
                 outgoing_foreign_keys = await db.foreign_keys_for_table(through_table)
                 fk_to_us = next(
@@ -185,6 +203,17 @@ class Filter:
         raise NotImplementedError
 
 
+def _coerce_numeric_filter_value(value):
+    try:
+        return int(value)
+    except ValueError:
+        try:
+            converted = float(value)
+        except ValueError:
+            return value
+        return converted if math.isfinite(converted) else value
+
+
 class TemplatedFilter(Filter):
     def __init__(
         self,
@@ -206,8 +235,8 @@ class TemplatedFilter(Filter):
 
     def where_clause(self, table, column, value, param_counter):
         converted = self.format.format(value)
-        if self.numeric and converted.isdigit():
-            converted = int(converted)
+        if self.numeric:
+            converted = _coerce_numeric_filter_value(converted)
         if self.no_argument:
             kwargs = {"c": _quote_sqlite_identifier(column)}
             converted = None

datasette/handle_exception.py

@@ -59,6 +59,10 @@ def handle_exception(datasette, request, exception):
             body = dict(info)
             body.update(error_body(plain_message or message, status))
             return Response.json(body, status=status, headers=headers)
+        if request.path.split("?")[0].endswith(".csv"):
+            return Response.text(
+                plain_message or message, status=status, headers=headers
+            )
         info.update(
             {
                 "ok": False,

datasette/hookspecs.py

@@ -9,6 +9,11 @@ def startup(datasette):
     """Fires directly after Datasette first starts running"""
 
 
+@hookspec
+def shutdown(datasette):
+    """Called once when the Datasette server is shutting down"""
+
+
 @hookspec
 def asgi_wrapper(datasette):
     """Returns an ASGI middleware callable to wrap our ASGI application with"""
@@ -45,7 +50,7 @@ def extra_body_script(
 def extra_template_vars(
     template, database, table, columns, view_name, request, datasette
 ):
-    """Extra template variables to be made available to the template - can return dict or callable or awaitable"""
+    """Extra template variables to be made available to the template - can return dict, None, callable or awaitable"""
 
 
 @hookspec

datasette/permissions.py

@@ -3,6 +3,10 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Any, NamedTuple
 
+_SQLITE_IDENTIFIER_CASE = str.maketrans(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
+)
+
 # Context variable to track when permission checks should be skipped
 _skip_permission_checks = contextvars.ContextVar(
     "skip_permission_checks", default=False
@@ -49,6 +53,15 @@ class Resource(ABC):
     # Class-level metadata (subclasses must define these)
     name: str = None  # e.g., "table", "database", "model"
     parent_class: type["Resource"] | None = None  # e.g., DatabaseResource for tables
+    case_insensitive_child: bool = False
+
+    @classmethod
+    def normalize_child(cls, child: str | None) -> str | None:
+        """Return a comparison key without changing the resource's display name."""
+        if cls.case_insensitive_child and child is not None:
+            # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold.
+            return child.translate(_SQLITE_IDENTIFIER_CASE)
+        return child
 
     # Instance-level optional extra attributes
     reasons: list[str] | None = None
@@ -146,6 +159,11 @@ class Action:
     resource_class: type[Resource] | None = None
     also_requires: str | None = None  # Optional action name that must also be allowed
 
+    def normalize_child(self, child: str | None) -> str | None:
+        if self.resource_class is None:
+            return child
+        return self.resource_class.normalize_child(child)
+
     @property
     def takes_parent(self) -> bool:
         """

datasette/plugins.py

@@ -18,6 +18,7 @@ DEFAULT_PLUGINS = (
     "datasette.actor_auth_cookie",
     "datasette.default_permissions",
     "datasette.default_permissions.tokens",
+    "datasette.default_permissions.sqlite_statistics",
     "datasette.default_actions",
     "datasette.default_column_types",
     "datasette.default_magic_parameters",

datasette/resources.py

@@ -25,6 +25,7 @@ class TableResource(Resource):
 
     name = "table"
     parent_class = DatabaseResource
+    case_insensitive_child = True
 
     def __init__(self, database: str, table: str):
         super().__init__(parent=database, child=table)

datasette/static/column-chooser.js

@@ -1,7 +1,9 @@
+let columnChooserInstanceCounter = 0;
+
 class ColumnChooser extends HTMLElement {
   constructor() {
     super();
-    this.attachShadow({ mode: "open" });
+    this.titleId = `column-chooser-title-${++columnChooserInstanceCounter}`;
 
     // State
     this._items = [];
@@ -26,375 +28,60 @@ class ColumnChooser extends HTMLElement {
     // Bound handlers
     this._onMove = this._onMove.bind(this);
     this._onUp = this._onUp.bind(this);
-
-    this.shadowRoot.innerHTML = `
-      <style>
-        :host {
-          --ink: #0f0f0f;
-          --paper: #eef6ff;
-          --muted: #6b6b6b;
-          --rule: #d8e6f5;
-          --accent: #1a56db;
-          --accent-light: #e8effd;
-          --card: #ffffff;
   }
 
-        * { box-sizing: border-box; margin: 0; padding: 0; }
-
-        dialog {
-          border: none;
-          border-radius: var(--modal-border-radius, 0.75rem);
-          padding: 0;
-          margin: auto;
-          width: 100%;
-          max-width: 420px;
-          max-height: min(640px, calc(100vh - 32px));
-          box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
-          animation: slideIn var(--modal-animation-duration, 0.2s) ease-out;
-          overflow: hidden;
-          font-family: system-ui, -apple-system, sans-serif;
-          background: var(--card);
-          -webkit-user-select: none;
-          -webkit-touch-callout: none;
-          -webkit-tap-highlight-color: transparent;
-        }
-
-        dialog[open] {
-          display: flex;
-          flex-direction: column;
-          height: min(640px, calc(100vh - 32px));
-        }
-
-        dialog::backdrop {
-          background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
-          backdrop-filter: var(--modal-backdrop-blur, blur(4px));
-          -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
-          animation: fadeIn var(--modal-animation-duration, 0.2s) ease-out;
-        }
-
-        @keyframes slideIn {
-          from {
-            opacity: 0;
-            transform: translateY(-20px) scale(0.95);
-          }
-          to {
-            opacity: 1;
-            transform: translateY(0) scale(1);
-          }
-        }
-
-        @keyframes fadeIn {
-          from { opacity: 0; }
-          to { opacity: 1; }
-        }
-
-        .modal-header {
-          padding: 20px 24px 16px;
-          border-bottom: 1px solid var(--rule);
-          display: flex;
-          align-items: center;
-          justify-content: space-between;
-          flex-shrink: 0;
-        }
-
-        .modal-title {
-          font-size: 1rem;
-          font-weight: 600;
-        }
-
-        .modal-meta {
-          font-family: ui-monospace, monospace;
-          font-size: 0.7rem;
-          color: var(--muted);
-          background: var(--paper);
-          padding: 3px 9px;
-          border-radius: 20px;
-        }
-
-        .list-toolbar {
-          padding: 6px 24px;
-          border-bottom: 1px solid var(--rule);
-          display: flex;
-          gap: 12px;
-          flex-shrink: 0;
-        }
-
-        .list-toolbar button {
-          background: var(--accent-light);
-          border: 1px solid var(--rule);
-          border-radius: 4px;
-          font-family: inherit;
-          font-size: 0.75rem;
-          color: var(--accent);
-          cursor: pointer;
-          padding: 3px 10px;
-          transition: background 0.12s, color 0.12s;
-        }
-        .list-toolbar button:hover { background: var(--accent); color: white; }
-
-        .list-wrap {
-          flex: 1;
-          overflow-y: auto;
-          overflow-x: hidden;
-          position: relative;
-          overscroll-behavior: contain;
-          -webkit-overflow-scrolling: touch;
-        }
-
-        .list-wrap::before,
-        .list-wrap::after {
-          content: '';
-          position: sticky;
-          display: block;
-          left: 0; right: 0;
-          height: 20px;
-          pointer-events: none;
-          z-index: 5;
-          transition: opacity 0.2s;
-        }
-        .list-wrap::before {
-          top: 0;
-          background: linear-gradient(to bottom, rgba(255,255,255,0.9), transparent);
-        }
-        .list-wrap::after {
-          bottom: 0;
-          background: linear-gradient(to top, rgba(255,255,255,0.9), transparent);
-          margin-top: -20px;
-        }
-
-        .scroll-zone {
-          position: absolute;
-          left: 0; right: 0;
-          height: 72px;
-          pointer-events: none;
-          z-index: 10;
-        }
-        .scroll-zone-top { top: 0; }
-        .scroll-zone-bot { bottom: 0; }
-
-        .drag-list {
-          list-style: none;
-          padding: 4px 0;
-        }
-
-        .drag-item {
-          display: flex;
-          align-items: center;
-          background: white;
-          border-bottom: 1px solid var(--rule);
-          user-select: none;
-          -webkit-user-select: none;
-          -webkit-touch-callout: none;
-          position: relative;
-          transition: background 0.08s;
-        }
-
-        .drag-item:last-child { border-bottom: none; }
-
-        .drag-handle {
-          display: flex;
-          align-items: center;
-          justify-content: center;
-          width: 48px;
-          height: 48px;
-          flex-shrink: 0;
-          cursor: grab;
-          color: #c8c4bc;
-          touch-action: none;
-          transition: color 0.15s;
-        }
-
-        .drag-handle:hover { color: var(--accent); }
-        .drag-handle svg { pointer-events: none; display: block; }
-
-        .drag-item-content {
-          display: flex;
-          align-items: center;
-          flex: 1;
-          min-width: 0;
-          cursor: pointer;
-        }
-
-        .drag-item-check {
-          display: flex;
-          align-items: center;
-          width: 32px;
-          height: 48px;
-          flex-shrink: 0;
-        }
-
-        .drag-item-check input[type="checkbox"] {
-          width: 16px;
-          height: 16px;
-          accent-color: var(--accent);
-          cursor: pointer;
-        }
-
-        .drag-item-label {
-          flex: 1;
-          font-size: 0.9rem;
-          line-height: 48px;
-          padding-right: 16px;
-          white-space: nowrap;
-          overflow: hidden;
-          text-overflow: ellipsis;
-          cursor: default;
-        }
-
-        .drag-item.is-dragging {
-          opacity: 0;
-        }
-
-        .drop-indicator {
-          position: absolute;
-          left: 48px;
-          right: 0;
-          height: 2px;
-          background: var(--accent);
-          border-radius: 99px;
-          pointer-events: none;
-          z-index: 20;
-          display: none;
-        }
-        .drop-indicator.top { top: -1px; display: block; }
-        .drop-indicator.bottom { bottom: -1px; display: block; }
-
-        .drag-ghost {
-          position: fixed;
-          pointer-events: none;
-          z-index: 9999;
-          background: white;
-          border-radius: 6px;
-          box-shadow: 0 8px 32px rgba(0,0,0,0.18), 0 2px 8px rgba(0,0,0,0.1);
-          display: flex;
-          align-items: center;
-          border: 1.5px solid var(--accent-light);
-          opacity: 0.97;
-          will-change: transform;
-          font-family: system-ui, -apple-system, sans-serif;
-        }
-
-        .scroll-pulse {
-          position: absolute;
-          left: 50%;
-          transform: translateX(-50%);
-          width: 32px;
-          height: 32px;
-          border-radius: 50%;
-          background: var(--accent);
-          opacity: 0;
-          pointer-events: none;
-          z-index: 10;
-          transition: opacity 0.15s;
-        }
-        .scroll-pulse.top { top: 8px; }
-        .scroll-pulse.bot { bottom: 8px; }
-        .scroll-pulse.active {
-          opacity: 0.18;
-          animation: pulse 0.8s ease-in-out infinite;
-        }
-
-        @keyframes pulse {
-          0%, 100% { transform: translateX(-50%) scale(1); opacity: 0.18; }
-          50% { transform: translateX(-50%) scale(1.5); opacity: 0.07; }
-        }
-
-        .modal-footer {
-          padding: 14px 20px;
-          border-top: 1px solid var(--rule);
-          display: flex;
-          align-items: center;
-          gap: 10px;
-          flex-shrink: 0;
-          background: var(--paper);
-        }
-
-        .footer-info {
-          flex: 1;
-          font-family: ui-monospace, monospace;
-          font-size: 0.68rem;
-          color: var(--muted);
-        }
-
-        .btn {
-          border: none;
-          border-radius: 5px;
-          padding: 9px 20px;
-          font-size: 0.85rem;
-          font-weight: 500;
-          cursor: pointer;
-          touch-action: manipulation;
-          font-family: inherit;
-          transition: background 0.12s;
-        }
-
-        .btn-primary {
-          background: var(--accent);
-          color: white;
-        }
-        .btn-primary:hover { background: #1448c0; }
-
-        .btn-ghost {
-          background: transparent;
-          color: var(--muted);
-          border: 1px solid var(--rule);
-        }
-        .btn-ghost:hover { background: var(--rule); color: var(--ink); }
-
-        .list-wrap::-webkit-scrollbar { width: 5px; }
-        .list-wrap::-webkit-scrollbar-track { background: transparent; }
-        .list-wrap::-webkit-scrollbar-thumb { background: var(--rule); border-radius: 99px; }
-
-        input, textarea { -webkit-user-select: auto; user-select: auto; }
-      </style>
-
-      <dialog aria-labelledby="modalTitle">
+  connectedCallback() {
+    if (this._modal) return;
+    this.innerHTML = `
+      <datasette-modal><dialog aria-labelledby="${this.titleId}">
           <div class="modal-header">
-            <span class="modal-title" id="modalTitle">Choose columns</span>
-            <span class="modal-meta" id="selectedCount"></span>
+            <span class="modal-title" id="${this.titleId}">Choose columns</span>
+            <span class="modal-meta"></span>
           </div>
           <div class="list-toolbar">
-            <button id="selectAllBtn">Select all</button>
-            <button id="deselectAllBtn">Deselect all</button>
+            <button class="select-all">Select all</button>
+            <button class="deselect-all">Deselect all</button>
           </div>
-          <div class="list-wrap" id="listWrap">
-            <div class="scroll-pulse top" id="pulseTop"></div>
-            <div class="scroll-pulse bot" id="pulseBot"></div>
-            <ul class="drag-list" id="dragList"></ul>
+          <div class="modal-body list-wrap">
+            <div class="scroll-pulse top"></div>
+            <div class="scroll-pulse bot"></div>
+            <ul class="drag-list"></ul>
           </div>
           <div class="modal-footer">
-            <span class="footer-info" id="footerInfo"></span>
-            <button class="btn btn-ghost" id="cancelBtn">Cancel</button>
-            <button class="btn btn-primary" id="applyBtn">Apply</button>
+            <span class="footer-info"></span>
+            <button class="modal-btn modal-btn-ghost">Cancel</button>
+            <button class="modal-btn modal-btn-primary">Apply</button>
           </div>
-      </dialog>
+      </dialog></datasette-modal>
     `;
 
     // DOM refs
-    this._dialog = this.shadowRoot.querySelector("dialog");
-    this._listWrap = this.shadowRoot.getElementById("listWrap");
-    this._dragList = this.shadowRoot.getElementById("dragList");
-    this._pulseTop = this.shadowRoot.getElementById("pulseTop");
-    this._pulseBot = this.shadowRoot.getElementById("pulseBot");
-    this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn");
-    this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn");
-    this._cancelBtn = this.shadowRoot.getElementById("cancelBtn");
-    this._applyBtn = this.shadowRoot.getElementById("applyBtn");
-    this._countEl = this.shadowRoot.getElementById("selectedCount");
-    this._footerEl = this.shadowRoot.getElementById("footerInfo");
+    this._modal = this.querySelector("datasette-modal");
+    this._listWrap = this.querySelector(".list-wrap");
+    this._dragList = this.querySelector(".drag-list");
+    this._pulseTop = this.querySelector(".scroll-pulse.top");
+    this._pulseBot = this.querySelector(".scroll-pulse.bot");
+    this._selectAllBtn = this.querySelector(".select-all");
+    this._deselectAllBtn = this.querySelector(".deselect-all");
+    this._cancelBtn = this.querySelector(".modal-btn-ghost");
+    this._applyBtn = this.querySelector(".modal-btn-primary");
+    this._countEl = this.querySelector(".modal-meta");
+    this._footerEl = this.querySelector(".footer-info");
 
     // Event listeners
     this._selectAllBtn.addEventListener("click", () => this._selectAll());
     this._deselectAllBtn.addEventListener("click", () => this._deselectAll());
-    this._cancelBtn.addEventListener("click", () => this._close());
+    this._cancelBtn.addEventListener("click", () =>
+      this._modal.requestClose("cancel"),
+    );
     this._applyBtn.addEventListener("click", () => this._apply());
-    this._dialog.addEventListener("click", (e) => {
-      if (e.target === this._dialog) this._close();
-    });
-    this._dialog.addEventListener("cancel", (e) => {
-      e.preventDefault();
-      this._close();
-    });
+    this._modal.beforeClose = () => {
+      this._items = this._savedItems ? [...this._savedItems] : this._items;
+      this._checked = this._savedChecked
+        ? new Set(this._savedChecked)
+        : this._checked;
+      return true;
+    };
   }
 
   /**
@@ -414,19 +101,11 @@ class ColumnChooser extends HTMLElement {
     this._savedChecked = new Set(this._checked);
 
     this._render();
-    this._dialog.showModal();
+    this._modal.show();
   }
 
   // ── Internal methods ──
 
-  _close() {
-    this._items = this._savedItems ? [...this._savedItems] : this._items;
-    this._checked = this._savedChecked
-      ? new Set(this._savedChecked)
-      : this._checked;
-    this._dialog.close();
-  }
-
   _selectAll() {
     this._items.forEach((col) => this._checked.add(col));
     this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => {
@@ -445,7 +124,7 @@ class ColumnChooser extends HTMLElement {
 
   _apply() {
     const selected = this._items.filter((col) => this._checked.has(col));
-    this._dialog.close();
+    this._modal.close();
     if (this._onApply) {
       this._onApply(selected);
     }
@@ -472,11 +151,13 @@ class ColumnChooser extends HTMLElement {
           <span class="drag-item-check">
             <input type="checkbox" ${this._checked.has(col) ? "checked" : ""}>
           </span>
-          <span class="drag-item-label">${col}</span>
+          <span class="drag-item-label"></span>
         </label>
         <div class="drop-indicator"></div>
       `;
 
+      li.querySelector(".drag-item-label").textContent = col;
+
       li.querySelector("input").addEventListener("change", (e) => {
         e.target.checked ? this._checked.add(col) : this._checked.delete(col);
         this._updateCounts();
@@ -509,7 +190,7 @@ class ColumnChooser extends HTMLElement {
     this._ghostOffX = e.clientX - rect.left;
     this._ghostOffY = e.clientY - rect.top;
 
-    // Build ghost inside shadow DOM
+    // Keep the drag preview inside the dialog so it stays above the backdrop.
     this._ghost = document.createElement("div");
     this._ghost.className = "drag-ghost";
     this._ghost.style.width = rect.width + "px";
@@ -518,7 +199,7 @@ class ColumnChooser extends HTMLElement {
     this._ghost.querySelector(".drop-indicator")?.remove();
     const h = this._ghost.querySelector(".drag-handle");
     if (h) h.style.color = "var(--accent)";
-    this.shadowRoot.appendChild(this._ghost);
+    this._modal.dialog.appendChild(this._ghost);
 
     srcEl.classList.add("is-dragging");
     this._positionGhost(e.clientX, e.clientY);

datasette/static/edit-tools.js

@@ -915,6 +915,7 @@ function showTableCreateDialogError(state, message) {
 
 function setTableCreateDialogSaving(state, isSaving) {
   state.isSaving = isSaving;
+  state.modal.busy = isSaving;
   state.columnList
     .querySelectorAll("input, select, button")
     .forEach(function (control) {
@@ -2043,8 +2044,7 @@ async function createTableFromDataPreview(state) {
     var tableUrl =
       responseData.table_url ||
       fallbackTableUrl(responseData.table || payload.table);
-    state.shouldRestoreFocus = false;
-    state.dialog.close();
+    state.modal.close({ restoreFocus: false });
     if (tableUrl) {
       location.href = tableUrl;
     } else {
@@ -2118,8 +2118,7 @@ async function saveTableCreateDialog(state) {
     var tableUrl =
       responseData.table_url ||
       fallbackTableUrl(responseData.table || payload.table);
-    state.shouldRestoreFocus = false;
-    state.dialog.close();
+    state.modal.close({ restoreFocus: false });
     if (tableUrl) {
       location.href = tableUrl;
     } else {
@@ -2141,18 +2140,6 @@ function confirmDiscardTableCreateChanges(state) {
   return window.confirm("Discard this new table?");
 }
 
-function closeTableCreateDialogIfConfirmed(state) {
-  if (!state || state.isSaving) {
-    return false;
-  }
-  if (!confirmDiscardTableCreateChanges(state)) {
-    return false;
-  }
-  state.shouldRestoreFocus = true;
-  state.dialog.close();
-  return true;
-}
-
 function ensureTableCreateDialog(manager) {
   if (tableCreateDialogState) {
     return tableCreateDialogState;
@@ -2161,7 +2148,8 @@ function ensureTableCreateDialog(manager) {
     return null;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.id = TABLE_CREATE_DIALOG_ID;
   dialog.className = "table-create-dialog";
   dialog.setAttribute("aria-labelledby", "table-create-title");
@@ -2171,7 +2159,7 @@ function ensureTableCreateDialog(manager) {
     </div>
     <form class="table-create-form" method="post" novalidate>
       <p class="table-create-error" id="table-create-error" role="alert" tabindex="-1" hidden></p>
-      <div class="table-create-fields">
+      <div class="modal-body table-create-fields">
         <div class="table-create-field">
           <label class="table-create-label" for="table-create-name">Table name</label>
           <input class="table-create-input table-create-table-name" id="table-create-name" type="text" name="table" required autocomplete="off">
@@ -2198,14 +2186,15 @@ function ensureTableCreateDialog(manager) {
       <div class="modal-footer">
         <a href="#" class="table-create-mode-link table-create-from-data">Create table from data</a>
         <a href="#" class="table-create-mode-link table-create-manual" hidden>Create table manually</a>
-        <button type="button" class="btn btn-ghost table-create-cancel">Cancel</button>
-        <button type="submit" class="btn btn-primary table-create-save">Create table</button>
+        <button type="button" class="modal-btn modal-btn-ghost table-create-cancel">Cancel</button>
+        <button type="submit" class="modal-btn modal-btn-primary table-create-save">Create table</button>
       </div>
     </form>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   tableCreateDialogState = {
+    modal: modal,
     dialog: dialog,
     form: dialog.querySelector(".table-create-form"),
     title: dialog.querySelector(".modal-title"),
@@ -2225,8 +2214,6 @@ function ensureTableCreateDialog(manager) {
     manualCreateLink: dialog.querySelector(".table-create-manual"),
     cancelButton: dialog.querySelector(".table-create-cancel"),
     saveButton: dialog.querySelector(".table-create-save"),
-    currentButton: null,
-    shouldRestoreFocus: true,
     isSaving: false,
     mode: "manual",
     dataPreviewRows: null,
@@ -2266,7 +2253,7 @@ function ensureTableCreateDialog(manager) {
       tableCreateDialogState.dataTextarea.focus();
       return;
     }
-    closeTableCreateDialogIfConfirmed(tableCreateDialogState);
+    modal.requestClose("cancel");
   });
 
   tableCreateDialogState.createFromDataLink.addEventListener(
@@ -2364,36 +2351,14 @@ function ensureTableCreateDialog(manager) {
     updateTableCreateDialogButtons(tableCreateDialogState);
   });
 
-  dialog.addEventListener("click", function (ev) {
-    if (ev.target === dialog) {
-      closeTableCreateDialogIfConfirmed(tableCreateDialogState);
-    }
-  });
-
-  dialog.addEventListener("keydown", function (ev) {
-    if (ev.key !== "Escape") {
-      return;
-    }
-    ev.preventDefault();
-    closeTableCreateDialogIfConfirmed(tableCreateDialogState);
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    ev.preventDefault();
-    closeTableCreateDialogIfConfirmed(tableCreateDialogState);
-  });
+  modal.beforeClose = function (source) {
+    return confirmDiscardTableCreateChanges(tableCreateDialogState);
+  };
 
   dialog.addEventListener("close", function () {
     var state = tableCreateDialogState;
     clearTableCreateDialogError(state);
     setTableCreateDialogSaving(state, false);
-    if (
-      state.shouldRestoreFocus &&
-      state.currentButton &&
-      document.contains(state.currentButton)
-    ) {
-      state.currentButton.focus();
-    }
   });
 
   return tableCreateDialogState;
@@ -2414,15 +2379,12 @@ function openTableCreateDialog(button, manager) {
     menu.open = false;
   }
   state.manager = manager;
-  state.currentButton = button;
-  state.shouldRestoreFocus = true;
+
   state.title.textContent = "Create a table in " + data.databaseName;
   clearTableCreateDialogError(state);
   resetTableCreateDialog(state);
   loadTableCreateForeignKeyTargets(state);
-  if (!state.dialog.open) {
-    state.dialog.showModal();
-  }
+  state.modal.show({ returnFocusTo: button });
   state.tableName.focus();
 }
 
@@ -2448,6 +2410,7 @@ function initTableCreateActions(manager) {
 
 function setRowDeleteDialogBusy(state, isBusy) {
   state.isBusy = isBusy;
+  state.modal.busy = isBusy;
   state.confirmButton.disabled = isBusy;
   state.cancelButton.disabled = isBusy;
   state.confirmButton.textContent = isBusy ? "Deleting..." : "Delete row";
@@ -2694,6 +2657,7 @@ function showTableAlterDialogError(state, message) {
 
 function setTableAlterDialogSaving(state, isSaving) {
   state.isSaving = isSaving;
+  state.modal.busy = isSaving;
   state.cancelButton.disabled = isSaving;
   state.addColumnButton.disabled = isSaving;
   state.backButton.disabled = isSaving;
@@ -3829,8 +3793,7 @@ async function applyTableAlterChanges(state, result) {
       result.columnTypeAssignments || [],
       tableUrl,
     );
-    state.shouldRestoreFocus = false;
-    state.dialog.close();
+    state.modal.close({ restoreFocus: false });
     if (tableAlterResultRenamesTable(result) && tableUrl) {
       window.location.href = tableUrl;
     } else {
@@ -3891,8 +3854,7 @@ async function dropTableFromAlterDialog(state) {
     if (!response.ok || (responseData && responseData.ok === false)) {
       throw rowMutationRequestError(response, responseData);
     }
-    state.shouldRestoreFocus = false;
-    state.dialog.close();
+    state.modal.close({ restoreFocus: false });
     window.location.href = tableAlterDatabaseUrl() || "/";
   } catch (error) {
     setTableAlterDialogSaving(state, false);
@@ -3928,27 +3890,6 @@ function confirmDiscardTableAlterChanges(state) {
   return window.confirm("Discard table changes?");
 }
 
-function closeTableAlterDialogIfConfirmed(state) {
-  if (!state || state.isSaving) {
-    return false;
-  }
-  if (!confirmDiscardTableAlterChanges(state)) {
-    return false;
-  }
-  state.shouldRestoreFocus = true;
-  state.dialog.close();
-  return true;
-}
-
-function closeTableAlterDialog(state) {
-  if (!state || state.isSaving) {
-    return false;
-  }
-  state.shouldRestoreFocus = true;
-  state.dialog.close();
-  return true;
-}
-
 function ensureTableAlterDialog(manager) {
   if (tableAlterDialogState) {
     return tableAlterDialogState;
@@ -3957,7 +3898,8 @@ function ensureTableAlterDialog(manager) {
     return null;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.id = TABLE_ALTER_DIALOG_ID;
   dialog.className = "table-alter-dialog";
   dialog.setAttribute("aria-labelledby", "table-alter-title");
@@ -3967,7 +3909,7 @@ function ensureTableAlterDialog(manager) {
     </div>
     <form class="table-alter-form" method="post" novalidate>
       <p class="table-alter-error" id="table-alter-error" role="alert" tabindex="-1" hidden></p>
-      <div class="table-alter-fields">
+      <div class="modal-body table-alter-fields">
         <div class="table-alter-columns">
           <div class="table-alter-column-headings" aria-hidden="true">
             <span>Column</span>
@@ -3986,18 +3928,19 @@ function ensureTableAlterDialog(manager) {
           </div>
         </details>
       </div>
-      <div class="table-alter-review" hidden></div>
+      <div class="modal-body table-alter-review" hidden></div>
       <div class="modal-footer">
-        <button type="button" class="btn btn-danger table-alter-drop" hidden>Drop table</button>
-        <button type="button" class="btn btn-ghost table-alter-back" hidden>Back</button>
-        <button type="button" class="btn btn-ghost table-alter-cancel">Cancel</button>
-        <button type="submit" class="btn btn-primary table-alter-save">Review changes</button>
+        <button type="button" class="modal-btn modal-btn-danger table-alter-drop" hidden>Drop table</button>
+        <button type="button" class="modal-btn modal-btn-ghost table-alter-back" hidden>Back</button>
+        <button type="button" class="modal-btn modal-btn-ghost table-alter-cancel">Cancel</button>
+        <button type="submit" class="modal-btn modal-btn-primary table-alter-save">Review changes</button>
       </div>
     </form>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   tableAlterDialogState = {
+    modal: modal,
     dialog: dialog,
     form: dialog.querySelector(".table-alter-form"),
     title: dialog.querySelector(".modal-title"),
@@ -4012,8 +3955,6 @@ function ensureTableAlterDialog(manager) {
     dropButton: dialog.querySelector(".table-alter-drop"),
     cancelButton: dialog.querySelector(".table-alter-cancel"),
     saveButton: dialog.querySelector(".table-alter-save"),
-    currentButton: null,
-    shouldRestoreFocus: true,
     isSaving: false,
     initialSignature: "",
     originalTableName: "",
@@ -4055,7 +3996,7 @@ function ensureTableAlterDialog(manager) {
   });
 
   tableAlterDialogState.cancelButton.addEventListener("click", function () {
-    closeTableAlterDialog(tableAlterDialogState);
+    modal.requestClose("cancel");
   });
 
   tableAlterDialogState.dropButton.addEventListener("click", function () {
@@ -4076,36 +4017,17 @@ function ensureTableAlterDialog(manager) {
     }
   });
 
-  dialog.addEventListener("click", function (ev) {
-    if (ev.target === dialog) {
-      closeTableAlterDialogIfConfirmed(tableAlterDialogState);
-    }
-  });
-
-  dialog.addEventListener("keydown", function (ev) {
-    if (ev.key !== "Escape") {
-      return;
-    }
-    ev.preventDefault();
-    closeTableAlterDialogIfConfirmed(tableAlterDialogState);
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    ev.preventDefault();
-    closeTableAlterDialogIfConfirmed(tableAlterDialogState);
-  });
+  modal.beforeClose = function (source) {
+    return (
+      source === "cancel" ||
+      confirmDiscardTableAlterChanges(tableAlterDialogState)
+    );
+  };
 
   dialog.addEventListener("close", function () {
     var state = tableAlterDialogState;
     clearTableAlterDialogError(state);
     setTableAlterDialogSaving(state, false);
-    if (
-      state.shouldRestoreFocus &&
-      state.currentButton &&
-      document.contains(state.currentButton)
-    ) {
-      state.currentButton.focus();
-    }
   });
 
   return tableAlterDialogState;
@@ -4126,8 +4048,7 @@ function openTableAlterDialog(button, manager) {
     menu.open = false;
   }
   state.manager = manager;
-  state.currentButton = button;
-  state.shouldRestoreFocus = true;
+
   state.title.textContent = "Alter table " + data.tableName;
   clearTableAlterDialogError(state);
   resetTableAlterDialog(state, data);
@@ -4137,9 +4058,7 @@ function openTableAlterDialog(button, manager) {
     tableAlterForeignKeyTargetsUrl(),
     { filterByType: false },
   );
-  if (!state.dialog.open) {
-    state.dialog.showModal();
-  }
+  state.modal.show({ returnFocusTo: button });
   var firstName = state.columnList.querySelector(".table-alter-column-name");
   if (firstName) {
     firstName.focus();
@@ -4442,7 +4361,8 @@ function ensureRowDeleteDialog(manager) {
     return null;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.id = ROW_DELETE_DIALOG_ID;
   dialog.className = "row-delete-dialog";
   dialog.setAttribute("aria-labelledby", "row-delete-title");
@@ -4454,13 +4374,14 @@ function ensureRowDeleteDialog(manager) {
     <p class="row-delete-message" id="row-delete-message">Delete row <span class="row-delete-id"></span>?</p>
     <p class="row-delete-error" role="alert" hidden></p>
     <div class="modal-footer">
-      <button type="button" class="btn btn-ghost row-delete-cancel">Cancel</button>
-      <button type="button" class="btn btn-primary row-delete-confirm">Delete row</button>
+      <button type="button" class="modal-btn modal-btn-ghost row-delete-cancel">Cancel</button>
+      <button type="button" class="modal-btn modal-btn-primary row-delete-confirm">Delete row</button>
     </div>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   rowDeleteDialogState = {
+    modal: modal,
     dialog: dialog,
     title: dialog.querySelector(".modal-title"),
     message: dialog.querySelector(".row-delete-message"),
@@ -4473,21 +4394,10 @@ function ensureRowDeleteDialog(manager) {
     currentPkPath: null,
     manager: manager,
     isBusy: false,
-    shouldRestoreFocus: true,
   };
 
   rowDeleteDialogState.cancelButton.addEventListener("click", function () {
-    if (!rowDeleteDialogState.isBusy) {
-      rowDeleteDialogState.shouldRestoreFocus = true;
-      dialog.close();
-    }
-  });
-
-  dialog.addEventListener("click", function (ev) {
-    if (ev.target === dialog && !rowDeleteDialogState.isBusy) {
-      rowDeleteDialogState.shouldRestoreFocus = true;
-      dialog.close();
-    }
+    modal.requestClose("cancel");
   });
 
   dialog.addEventListener("keydown", function (ev) {
@@ -4499,25 +4409,6 @@ function ensureRowDeleteDialog(manager) {
       if (!rowDeleteDialogState.isBusy) {
         rowDeleteDialogState.confirmButton.click();
       }
-      return;
-    }
-    if (ev.key !== "Escape") {
-      return;
-    }
-    if (rowDeleteDialogState.isBusy) {
-      ev.preventDefault();
-      return;
-    }
-    ev.preventDefault();
-    rowDeleteDialogState.shouldRestoreFocus = true;
-    dialog.close();
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    if (rowDeleteDialogState.isBusy) {
-      ev.preventDefault();
-    } else {
-      rowDeleteDialogState.shouldRestoreFocus = true;
     }
   });
 
@@ -4525,13 +4416,6 @@ function ensureRowDeleteDialog(manager) {
     var state = rowDeleteDialogState;
     clearRowDeleteDialogError(state);
     setRowDeleteDialogBusy(state, false);
-    if (
-      state.shouldRestoreFocus &&
-      state.currentButton &&
-      document.contains(state.currentButton)
-    ) {
-      state.currentButton.focus();
-    }
   });
 
   rowDeleteDialogState.confirmButton.addEventListener(
@@ -4558,8 +4442,7 @@ function ensureRowDeleteDialog(manager) {
           throw rowMutationRequestError(response, data);
         }
         if (data && data.redirect) {
-          state.shouldRestoreFocus = false;
-          state.dialog.close();
+          state.modal.close({ restoreFocus: false });
           location.href = data.redirect;
           return;
         }
@@ -4571,8 +4454,7 @@ function ensureRowDeleteDialog(manager) {
         var statusMessage = state.currentPkPath
           ? "Deleted row " + state.currentPkPath + "."
           : "Deleted row.";
-        state.shouldRestoreFocus = false;
-        state.dialog.close();
+        state.modal.close({ restoreFocus: false });
         state.currentRow.remove();
         showRowMutationStatus(state.manager, statusMessage, false);
         if (focusTarget && document.contains(focusTarget)) {
@@ -4601,11 +4483,9 @@ function openRowDeleteDialog(button, manager) {
   }
 
   state.manager = manager;
-  state.currentButton = button;
   state.currentRow = row;
   state.currentDeleteUrl = rowDeleteUrl(row);
   state.currentPkPath = rowDisplayLabel(row);
-  state.shouldRestoreFocus = true;
 
   clearRowDeleteDialogError(state);
   setRowDeleteDialogBusy(state, false);
@@ -4617,9 +4497,7 @@ function openRowDeleteDialog(button, manager) {
   );
   state.rowId.textContent = state.currentPkPath || "this row";
 
-  if (!state.dialog.open) {
-    state.dialog.showModal();
-  }
+  state.modal.show({ returnFocusTo: button });
   state.confirmButton.focus();
 }
 
@@ -5694,6 +5572,7 @@ function setRowEditDialogLoading(state, isLoading) {
 
 function setRowEditDialogSaving(state, isSaving) {
   state.isSaving = isSaving;
+  state.modal.busy = isSaving;
   updateRowEditDialogButtons(state);
 }
 
@@ -5911,18 +5790,6 @@ function confirmDiscardRowEditChanges(state) {
   return window.confirm(message);
 }
 
-function closeRowEditDialogIfConfirmed(state) {
-  if (!state || state.isSaving) {
-    return false;
-  }
-  if (!confirmDiscardRowEditChanges(state)) {
-    return false;
-  }
-  state.shouldRestoreFocus = true;
-  state.dialog.close();
-  return true;
-}
-
 function setRowInsertDialogTitle(state) {
   var insertData = tableInsertData() || {};
   var title = rowEditIsMultipleInsert(state)
@@ -6748,38 +6615,6 @@ async function insertBulkPreviewRows(state) {
   }
 }
 
-function scheduleCloseRowEditDialogIfConfirmed(state) {
-  // Fix for an issue in Safari where hitting Esc would show
-  // the confirm() prompt asking if state should be discarded
-  // but the Esc key press would then cancel that dialog too.
-  // Wait for keyup, then move the confirm() to a fresh timer tick.
-  if (!state || state.isSaving || state.isClosePending) {
-    return false;
-  }
-  if (!rowEditDialogHasChanges(state)) {
-    state.shouldRestoreFocus = true;
-    state.dialog.close();
-    return true;
-  }
-  state.isClosePending = true;
-  var closeAfterKeyup = function () {
-    if (!state.isClosePending) {
-      return;
-    }
-    state.isClosePending = false;
-    closeRowEditDialogIfConfirmed(state);
-  };
-  var onKeyup = function (ev) {
-    if (ev.key !== "Escape") {
-      return;
-    }
-    document.removeEventListener("keyup", onKeyup, true);
-    setTimeout(closeAfterKeyup, 0);
-  };
-  document.addEventListener("keyup", onKeyup, true);
-  return true;
-}
-
 function findDataRowElement(root, rowId) {
   var elements = root.querySelectorAll("[data-row]");
   for (var i = 0; i < elements.length; i += 1) {
@@ -6869,9 +6704,8 @@ async function saveRowEditDialog(state) {
     }
     var formValues = collectRowFormValues(state);
     if (state.mode === "edit" && !Object.keys(formValues).length) {
-      state.shouldRestoreFocus = true;
       hideRowMutationStatus();
-      state.dialog.close();
+      state.modal.close();
       return;
     }
     var payload =
@@ -6904,9 +6738,8 @@ async function saveRowEditDialog(state) {
         insertedRowData,
         insertData.primaryKeys || [],
       );
-      state.shouldRestoreFocus = false;
       if (!insertedRowId) {
-        state.dialog.close();
+        state.modal.close({ restoreFocus: false });
         var missingIdStatus = showRowMutationStatus(
           state.manager,
           "Inserted row. Refresh the page to see it.",
@@ -6922,7 +6755,7 @@ async function saveRowEditDialog(state) {
       try {
         insertedRow = await fetchUpdatedRowElement(state);
       } catch (_error) {
-        state.dialog.close();
+        state.modal.close({ restoreFocus: false });
         var refreshFailedStatus = showRowMutationStatus(
           state.manager,
           "Inserted row, but could not refresh the table row. Refresh the page to see it.",
@@ -6937,7 +6770,7 @@ async function saveRowEditDialog(state) {
           rowTitleLabel(insertedRow),
         );
         var addedRow = addInsertedRowToPage(insertedRow);
-        state.dialog.close();
+        state.modal.close({ restoreFocus: false });
         showRowMutationStatus(state.manager, insertedStatusMessage, false);
         if (addedRow) {
           var insertedFocusTarget =
@@ -6946,7 +6779,7 @@ async function saveRowEditDialog(state) {
           insertedFocusTarget.focus();
         }
       } else {
-        state.dialog.close();
+        state.modal.close({ restoreFocus: false });
         var filteredStatus = showRowMutationStatus(
           state.manager,
           "Inserted row. It does not match the current filters.",
@@ -6958,8 +6791,7 @@ async function saveRowEditDialog(state) {
     }
 
     if (isRowPage()) {
-      state.shouldRestoreFocus = false;
-      state.dialog.close();
+      state.modal.close({ restoreFocus: false });
       location.reload();
       return;
     }
@@ -6995,8 +6827,7 @@ async function saveRowEditDialog(state) {
       );
     }
 
-    state.shouldRestoreFocus = false;
-    state.dialog.close();
+    state.modal.close({ restoreFocus: false });
     if (focusTarget && document.contains(focusTarget)) {
       focusTarget.focus();
     }
@@ -7140,7 +6971,8 @@ function ensureRowEditDialog(manager) {
     return null;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.id = ROW_EDIT_DIALOG_ID;
   dialog.className = "row-edit-dialog";
   dialog.setAttribute("aria-labelledby", "row-edit-title");
@@ -7152,8 +6984,8 @@ function ensureRowEditDialog(manager) {
       <p class="row-edit-summary" id="row-edit-summary" hidden></p>
       <p class="row-edit-loading" role="status" aria-live="polite">Loading row...</p>
       <p class="row-edit-error" role="alert" tabindex="-1" hidden></p>
-      <div class="row-edit-fields"></div>
-      <div class="row-edit-bulk" hidden>
+      <div class="modal-body row-edit-fields"></div>
+      <div class="modal-body row-edit-bulk" hidden>
         <div class="row-edit-bulk-editor">
           <p class="row-edit-bulk-note"><label for="row-edit-bulk-textarea">Paste TSV, CSV, or JSON</label>. You can also <button type="button" class="button-as-link row-edit-bulk-open-file">open a file</button> or drop it onto this textarea</p>
           <input class="row-edit-bulk-file-input" type="file" accept=".csv,.tsv,.json,.txt,text/csv,text/tab-separated-values,application/json,text/plain" hidden>
@@ -7170,7 +7002,7 @@ function ensureRowEditDialog(manager) {
             </div>
           </div>
           <div class="row-edit-bulk-actions">
-            <button type="button" class="btn btn-ghost row-edit-copy-template"><span class="row-edit-copy-template-label-wide">Copy spreadsheet template</span><span class="row-edit-copy-template-label-narrow">Copy template</span></button>
+            <button type="button" class="modal-btn modal-btn-ghost row-edit-copy-template"><span class="row-edit-copy-template-label-wide">Copy spreadsheet template</span><span class="row-edit-copy-template-label-narrow">Copy template</span></button>
             <span class="row-edit-bulk-template-note"><span class="row-edit-bulk-template-note-wide">You can paste the template into Google Sheets or Excel.</span><span class="row-edit-bulk-template-note-narrow">Paste into Google Sheets or Excel</span></span>
           </div>
         </div>
@@ -7183,14 +7015,15 @@ function ensureRowEditDialog(manager) {
       <div class="modal-footer">
         <a href="#" class="row-edit-mode-link row-edit-bulk-insert" hidden>Insert multiple rows</a>
         <a href="#" class="row-edit-mode-link row-edit-single-insert" hidden>Insert single row</a>
-        <button type="button" class="btn btn-ghost row-edit-cancel">Cancel</button>
-        <button type="submit" class="btn btn-primary row-edit-save" disabled>Save</button>
+        <button type="button" class="modal-btn modal-btn-ghost row-edit-cancel">Cancel</button>
+        <button type="submit" class="modal-btn modal-btn-primary row-edit-save" disabled>Save</button>
       </div>
     </form>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   rowEditDialogState = {
+    modal: modal,
     dialog: dialog,
     form: dialog.querySelector(".row-edit-form"),
     title: dialog.querySelector(".modal-title"),
@@ -7221,7 +7054,6 @@ function ensureRowEditDialog(manager) {
     singleInsertLink: dialog.querySelector(".row-edit-single-insert"),
     cancelButton: dialog.querySelector(".row-edit-cancel"),
     saveButton: dialog.querySelector(".row-edit-save"),
-    currentButton: null,
     currentRow: null,
     currentRowId: null,
     currentPkPath: null,
@@ -7249,9 +7081,7 @@ function ensureRowEditDialog(manager) {
     manager: manager,
     isLoading: false,
     isSaving: false,
-    isClosePending: false,
     hasLoaded: false,
-    shouldRestoreFocus: true,
   };
 
   rowEditDialogState.form.addEventListener("submit", function (ev) {
@@ -7271,10 +7101,7 @@ function ensureRowEditDialog(manager) {
       rowEditDialogState.bulkInsertTextarea.focus();
       return;
     }
-    if (!rowEditDialogState.isSaving) {
-      rowEditDialogState.shouldRestoreFocus = true;
-      dialog.close();
-    }
+    modal.requestClose("cancel");
   });
 
   rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) {
@@ -7393,31 +7220,17 @@ function ensureRowEditDialog(manager) {
     },
   );
 
-  dialog.addEventListener("click", function (ev) {
-    if (ev.target === dialog) {
-      closeRowEditDialogIfConfirmed(rowEditDialogState);
-    }
-  });
-
-  dialog.addEventListener("keydown", function (ev) {
-    if (ev.key !== "Escape") {
-      return;
-    }
-    ev.preventDefault();
-    scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState);
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    ev.preventDefault();
-    scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState);
-  });
+  modal.beforeClose = function (source) {
+    return (
+      source === "cancel" || confirmDiscardRowEditChanges(rowEditDialogState)
+    );
+  };
 
   dialog.addEventListener("close", function () {
     var state = rowEditDialogState;
     var shouldReloadOnClose = state.shouldReloadOnClose;
     var redirectOnCloseUrl = state.redirectOnCloseUrl;
     state.loadId += 1;
-    state.isClosePending = false;
     state.bulkInsertLiveValidationError = null;
     state.shouldReloadOnClose = false;
     state.redirectOnCloseUrl = null;
@@ -7430,13 +7243,6 @@ function ensureRowEditDialog(manager) {
     destroyRowEditFields(state);
     setRowEditDialogLoading(state, false);
     setRowEditDialogSaving(state, false);
-    if (
-      state.shouldRestoreFocus &&
-      state.currentButton &&
-      document.contains(state.currentButton)
-    ) {
-      state.currentButton.focus();
-    }
     if (shouldReloadOnClose) {
       if (redirectOnCloseUrl) {
         location.href = redirectOnCloseUrl;
@@ -7461,7 +7267,6 @@ async function openRowEditDialog(button, manager) {
 
   state.manager = manager;
   state.mode = "edit";
-  state.currentButton = button;
   state.currentRow = row;
   state.currentRowId = row.getAttribute("data-row") || "";
   state.currentPkPath = rowDisplayLabel(row);
@@ -7478,7 +7283,7 @@ async function openRowEditDialog(button, manager) {
   } else {
     state.form.removeAttribute("action");
   }
-  state.shouldRestoreFocus = true;
+
   state.hasLoaded = false;
   state.loadId += 1;
   var loadId = state.loadId;
@@ -7497,9 +7302,7 @@ async function openRowEditDialog(button, manager) {
   state.summary.textContent = "";
   syncRowEditInsertModeUi(state);
 
-  if (!state.dialog.open) {
-    state.dialog.showModal();
-  }
+  state.modal.show({ returnFocusTo: button });
   state.cancelButton.focus();
 
   try {
@@ -7539,7 +7342,6 @@ function openRowInsertDialog(button, manager) {
 
   state.manager = manager;
   state.mode = "insert";
-  state.currentButton = button;
   state.currentRow = null;
   state.currentRowId = null;
   state.currentPkPath = null;
@@ -7554,7 +7356,7 @@ function openRowInsertDialog(button, manager) {
   state.shouldReloadOnClose = false;
   state.redirectOnCloseUrl = null;
   resetBulkInsertPreview(state);
-  state.shouldRestoreFocus = true;
+
   state.hasLoaded = false;
   state.loadId += 1;
 
@@ -7576,9 +7378,7 @@ function openRowInsertDialog(button, manager) {
   state.summary.textContent = "";
   syncRowEditInsertModeUi(state);
 
-  if (!state.dialog.open) {
-    state.dialog.showModal();
-  }
+  state.modal.show({ returnFocusTo: button });
   renderRowInsertFields(state, insertData);
 }
 

datasette/static/json-format-highlight-1.0.1.js

@@ -1,56 +0,0 @@
-/*
-https://github.com/luyilin/json-format-highlight
-From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js
-MIT Licensed
-*/
-(function (global, factory) {
-  typeof exports === "object" && typeof module !== "undefined"
-    ? (module.exports = factory())
-    : typeof define === "function" && define.amd
-      ? define(factory)
-      : (global.jsonFormatHighlight = factory());
-})(this, function () {
-  "use strict";
-
-  var defaultColors = {
-    keyColor: "dimgray",
-    numberColor: "lightskyblue",
-    stringColor: "lightcoral",
-    trueColor: "lightseagreen",
-    falseColor: "#f66578",
-    nullColor: "cornflowerblue",
-  };
-
-  function index(json, colorOptions) {
-    if (colorOptions === void 0) colorOptions = {};
-
-    if (!json) {
-      return;
-    }
-    if (typeof json !== "string") {
-      json = JSON.stringify(json, null, 2);
-    }
-    var colors = Object.assign({}, defaultColors, colorOptions);
-    json = json.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
-    return json.replace(
-      /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g,
-      function (match) {
-        var color = colors.numberColor;
-        if (/^"/.test(match)) {
-          color = /:$/.test(match) ? colors.keyColor : colors.stringColor;
-        } else {
-          color = /true/.test(match)
-            ? colors.trueColor
-            : /false/.test(match)
-              ? colors.falseColor
-              : /null/.test(match)
-                ? colors.nullColor
-                : color;
-        }
-        return '<span style="color: ' + color + '">' + match + "</span>";
-      },
-    );
-  }
-
-  return index;
-});

datasette/static/mobile-column-actions.js

@@ -66,7 +66,8 @@ function initMobileColumnActions(manager) {
     return;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.className = "mobile-column-actions-dialog";
   dialog.id = MOBILE_COLUMN_DIALOG_ID;
   dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID);
@@ -75,13 +76,13 @@ function initMobileColumnActions(manager) {
       <span class="modal-title" id="${MOBILE_COLUMN_DIALOG_TITLE_ID}">Column actions</span>
       <span class="modal-meta"></span>
     </div>
-    <div class="list-wrap mobile-column-list"></div>
+    <div class="modal-body list-wrap mobile-column-list"></div>
     <div class="modal-footer">
       <span class="footer-info">Tap a column to reveal actions.</span>
-      <button type="button" class="btn btn-ghost mobile-column-actions-done">Done</button>
+      <button type="button" class="modal-btn modal-btn-ghost mobile-column-actions-done">Done</button>
     </div>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   triggerButton.setAttribute("aria-haspopup", "dialog");
   triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID);
@@ -91,7 +92,6 @@ function initMobileColumnActions(manager) {
   var listWrap = dialog.querySelector(".mobile-column-list");
   var doneButton = dialog.querySelector(".mobile-column-actions-done");
   var expandedSectionId = null;
-  var shouldRestoreFocus = true;
 
   function updateExpandedSection() {
     Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => {
@@ -128,16 +128,7 @@ function initMobileColumnActions(manager) {
   }
 
   function closeDialog(options) {
-    options = options || {};
-    shouldRestoreFocus = options.restoreFocus !== false;
-    if (dialog.open) {
-      dialog.close();
-    } else {
-      triggerButton.setAttribute("aria-expanded", "false");
-      if (shouldRestoreFocus) {
-        triggerButton.focus();
-      }
-    }
+    modal.close(options);
   }
 
   function renderDialog() {
@@ -166,7 +157,8 @@ function initMobileColumnActions(manager) {
       topActions.className = "mobile-column-top-actions";
 
       var showAllColumns = document.createElement("a");
-      showAllColumns.className = "btn btn-ghost mobile-column-top-action";
+      showAllColumns.className =
+        "modal-btn modal-btn-ghost mobile-column-top-action";
       showAllColumns.href = manager.columnActions.showAllColumnsUrl();
       showAllColumns.textContent = "Show all columns";
 
@@ -265,9 +257,7 @@ function initMobileColumnActions(manager) {
     if (!renderDialog()) {
       return;
     }
-    if (!dialog.open) {
-      dialog.showModal();
-    }
+    modal.show({ returnFocusTo: triggerButton });
     triggerButton.setAttribute("aria-expanded", "true");
     var focusTarget =
       dialog.querySelector(".mobile-column-top-action") ||
@@ -288,22 +278,8 @@ function initMobileColumnActions(manager) {
     closeDialog();
   });
 
-  dialog.addEventListener("click", function (ev) {
-    if (ev.target === dialog) {
-      closeDialog();
-    }
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    ev.preventDefault();
-    closeDialog();
-  });
-
   dialog.addEventListener("close", function () {
     triggerButton.setAttribute("aria-expanded", "false");
-    if (shouldRestoreFocus) {
-      triggerButton.focus();
-    }
   });
 
   window.addEventListener("resize", function () {

datasette/static/modal.js

@@ -0,0 +1,167 @@
+// Shared lifecycle for native modal dialogs.
+(() => {
+  class DatasetteModal extends HTMLElement {
+    constructor() {
+      super();
+      this.beforeClose = null;
+      this._busy = false;
+      this._restoreFocus = true;
+      this._returnFocusTo = null;
+      this._escapeCleanup = null;
+      this._escapeTimer = null;
+    }
+
+    static create() {
+      const modal = document.createElement("datasette-modal");
+      modal.appendChild(document.createElement("dialog"));
+      return modal;
+    }
+
+    get dialog() {
+      return this.querySelector(":scope > dialog");
+    }
+
+    get busy() {
+      return this._busy;
+    }
+
+    set busy(value) {
+      this._busy = !!value;
+      if (this.dialog) {
+        this.dialog.setAttribute("aria-busy", String(this._busy));
+      }
+    }
+
+    connectedCallback() {
+      const dialog = this.dialog;
+      if (!dialog) return;
+      dialog.classList.add("datasette-modal");
+      this._listeners?.abort();
+      this._listeners = new AbortController();
+      const options = { signal: this._listeners.signal };
+      let backdropPointerDown = false;
+      const outside = (event) => {
+        const rect = dialog.getBoundingClientRect();
+        return (
+          event.target === dialog &&
+          (event.clientX < rect.left ||
+            event.clientX > rect.right ||
+            event.clientY < rect.top ||
+            event.clientY > rect.bottom)
+        );
+      };
+      dialog.addEventListener(
+        "pointerdown",
+        (event) => {
+          backdropPointerDown = outside(event);
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "click",
+        (event) => {
+          if (backdropPointerDown && outside(event))
+            this.requestClose("backdrop");
+          backdropPointerDown = false;
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "keydown",
+        (event) => {
+          if (event.key !== "Escape" || event.defaultPrevented) return;
+          // A nested native dialog or plugin picker gets first refusal.
+          if (event.target.closest("dialog") !== dialog) return;
+          event.preventDefault();
+          if (this.busy || this._escapeCleanup || this._escapeTimer !== null)
+            return;
+          // Safari can otherwise use this Escape press to cancel confirm() too.
+          // Only keyboard dismissals wait for keyup; native cancel events needn't.
+          const onKeyup = (up) => {
+            if (up.key !== "Escape") return;
+            this._escapeCleanup();
+            this._escapeCleanup = null;
+            this._escapeTimer = setTimeout(() => {
+              this._escapeTimer = null;
+              this.requestClose("escape");
+            }, 0);
+          };
+          this.ownerDocument.addEventListener("keyup", onKeyup, true);
+          this._escapeCleanup = () =>
+            this.ownerDocument.removeEventListener("keyup", onKeyup, true);
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "cancel",
+        (event) => {
+          if (event.target !== dialog) return;
+          event.preventDefault();
+          if (!this._escapeCleanup && this._escapeTimer === null)
+            this.requestClose("escape");
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "close",
+        (event) => {
+          if (event.target !== dialog || dialog.open) return;
+          this._clearPendingClose();
+          this.busy = false;
+          if (this._restoreFocus && this._returnFocusTo?.isConnected) {
+            // Menu actions may have become hidden while the dialog was open.
+            const details = this._returnFocusTo.closest("details:not([open])");
+            const target =
+              details?.querySelector("summary") || this._returnFocusTo;
+            target.focus({ preventScroll: true });
+          }
+          this._returnFocusTo = null;
+        },
+        options,
+      );
+    }
+
+    disconnectedCallback() {
+      this._listeners?.abort();
+      this._clearPendingClose();
+      this._returnFocusTo = null;
+      if (this.dialog?.open) this.dialog.close();
+      this.busy = false;
+    }
+
+    _clearPendingClose() {
+      this._escapeCleanup?.();
+      this._escapeCleanup = null;
+      clearTimeout(this._escapeTimer);
+      this._escapeTimer = null;
+    }
+
+    show({ returnFocusTo, initialFocus } = {}) {
+      const dialog = this.dialog;
+      if (!dialog.open) {
+        this._clearPendingClose();
+        this._returnFocusTo = returnFocusTo || this.ownerDocument.activeElement;
+        this._restoreFocus = true;
+        dialog.showModal();
+      }
+      if (typeof initialFocus === "function") initialFocus();
+      else initialFocus?.focus();
+    }
+
+    requestClose(source = "cancel") {
+      if (!this.dialog.open || this.busy) return false;
+      if (this.beforeClose && this.beforeClose(source) === false) return false;
+      this.close();
+      return true;
+    }
+
+    close({ restoreFocus = true } = {}) {
+      this._clearPendingClose();
+      this._restoreFocus = restoreFocus;
+      this.dialog.close();
+    }
+  }
+
+  customElements.define("datasette-modal", DatasetteModal);
+  window.DatasetteModal = DatasetteModal;
+})();

datasette/static/navigation-search.js

@@ -10,277 +10,22 @@ class NavigationSearch extends HTMLElement {
     this.recentHeadingId = `navigation-search-recent-${this.instanceId}`;
     this.statusId = `navigation-search-status-${this.instanceId}`;
     this.titleId = `navigation-search-title-${this.instanceId}`;
-    this.attachShadow({ mode: "open" });
     this.selectedIndex = -1;
     this.matches = [];
     this.renderedMatches = [];
     this.debounceTimer = null;
-    this.restoreFocusTarget = null;
-    this.shouldRestoreFocus = true;
+  }
 
+  connectedCallback() {
+    if (this._initialized) return;
+    this._initialized = true;
     this.render();
     this.setupEventListeners();
   }
 
   render() {
-    this.shadowRoot.innerHTML = `
-            <style>
-                :host {
-                    display: contents;
-                }
-
-                dialog {
-                    border: none;
-                    border-radius: var(--modal-border-radius, 0.75rem);
-                    padding: 0;
-                    max-width: 90vw;
-                    width: 600px;
-                    max-height: 80vh;
-                    box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
-                    animation: slideIn var(--modal-animation-duration, 0.2s) ease-out;
-                }
-
-                dialog::backdrop {
-                    background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
-                    backdrop-filter: var(--modal-backdrop-blur, blur(4px));
-                    -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
-                    animation: fadeIn var(--modal-animation-duration, 0.2s) ease-out;
-                }
-
-                @keyframes slideIn {
-                    from {
-                        opacity: 0;
-                        transform: translateY(-20px) scale(0.95);
-                    }
-                    to {
-                        opacity: 1;
-                        transform: translateY(0) scale(1);
-                    }
-                }
-
-                @keyframes fadeIn {
-                    from { opacity: 0; }
-                    to { opacity: 1; }
-                }
-
-                .search-container {
-                    display: flex;
-                    flex-direction: column;
-                }
-
-                .search-input-wrapper {
-                    padding: 1.25rem;
-                    border-bottom: 1px solid #e5e7eb;
-                    display: flex;
-                    gap: 0.5rem;
-                    align-items: center;
-                }
-
-                .search-input {
-                    width: 100%;
-                    flex: 1;
-                    min-width: 0;
-                    padding: 0.75rem 1rem;
-                    font-size: 1rem;
-                    border: 2px solid #e5e7eb;
-                    border-radius: 0.5rem;
-                    outline: none;
-                    transition: border-color 0.2s;
-                    box-sizing: border-box;
-                }
-
-                .search-input:focus {
-                    border-color: #2563eb;
-                }
-
-                .close-search {
-                    background: transparent;
-                    border: 1px solid transparent;
-                    border-radius: 0.375rem;
-                    color: #4b5563;
-                    cursor: pointer;
-                    flex: 0 0 auto;
-                    font: inherit;
-                    font-size: 1.5rem;
-                    height: 2.75rem;
-                    line-height: 1;
-                    width: 2.75rem;
-                }
-
-                .close-search:hover,
-                .close-search:focus {
-                    background-color: #f3f4f6;
-                    border-color: #d1d5db;
-                }
-
-                .results-container {
-                    overflow-y: auto;
-                    height: calc(80vh - 180px);
-                    padding: 0.5rem;
-                }
-
-                .results-list:empty {
-                    display: none;
-                }
-
-                .result-item {
-                    padding: 0.875rem 1rem;
-                    cursor: pointer;
-                    border-radius: 0.5rem;
-                    transition: background-color 0.15s;
-                    display: flex;
-                    align-items: center;
-                    gap: 0.75rem;
-                }
-
-                .result-item:hover {
-                    background-color: #f3f4f6;
-                }
-
-                .result-item.selected {
-                    background-color: #dbeafe;
-                }
-
-                .result-item > div {
-                    flex: 1;
-                    min-width: 0;
-                }
-
-                .jump-start-content {
-                    border-bottom: 1px solid #e5e7eb;
-                    margin-bottom: 0.5rem;
-                    padding: 0.5rem 0.5rem 1rem;
-                }
-
-                .jump-start-content:empty {
-                    display: none;
-                }
-
-                .result-name {
-                    font-weight: 500;
-                    color: #111827;
-                }
-
-                .result-label {
-                    font-size: 0.875rem;
-                    color: #4b5563;
-                }
-
-                .result-type {
-                    color: #4b5563;
-                    font-size: 0.75rem;
-                    font-weight: 600;
-                    text-transform: uppercase;
-                }
-
-                .result-url {
-                    font-size: 0.875rem;
-                    color: #6b7280;
-                }
-
-                .result-description {
-                    color: #374151;
-                    display: -webkit-box;
-                    font-size: 0.8125rem;
-                    line-height: 1.35;
-                    margin-top: 0.35rem;
-                    overflow: hidden;
-                    -webkit-box-orient: vertical;
-                    -webkit-line-clamp: 2;
-                }
-
-                .results-heading {
-                    color: #4b5563;
-                    font-size: 0.75rem;
-                    font-weight: 600;
-                    letter-spacing: 0;
-                    padding: 0.5rem 1rem 0.25rem;
-                    text-transform: uppercase;
-                }
-
-                .recent-actions {
-                    padding: 0.25rem 1rem 0.75rem;
-                }
-
-                .clear-recent {
-                    background: transparent;
-                    border: 0;
-                    color: #2563eb;
-                    cursor: pointer;
-                    font: inherit;
-                    font-size: 0.875rem;
-                    padding: 0;
-                }
-
-                .clear-recent:hover {
-                    text-decoration: underline;
-                }
-
-                .no-results {
-                    padding: 2rem;
-                    text-align: center;
-                    color: #6b7280;
-                }
-
-                .hint-text {
-                    padding: 0.75rem 1.25rem;
-                    font-size: 0.875rem;
-                    color: #6b7280;
-                    border-top: 1px solid #e5e7eb;
-                    display: flex;
-                    gap: 1rem;
-                    flex-wrap: wrap;
-                }
-
-                .hint-text kbd {
-                    background: #f3f4f6;
-                    padding: 0.125rem 0.375rem;
-                    border-radius: 0.25rem;
-                    font-size: 0.75rem;
-                    border: 1px solid #d1d5db;
-                    font-family: monospace;
-                }
-
-                .visually-hidden {
-                    border: 0;
-                    clip: rect(0 0 0 0);
-                    height: 1px;
-                    margin: -1px;
-                    overflow: hidden;
-                    padding: 0;
-                    position: absolute;
-                    white-space: nowrap;
-                    width: 1px;
-                }
-
-                /* Mobile optimizations */
-                @media (max-width: 640px) {
-                    dialog {
-                        width: 95vw;
-                        max-height: 85vh;
-                        border-radius: 0.5rem;
-                    }
-
-                    .search-input-wrapper {
-                        padding: 1rem;
-                    }
-
-                    .search-input {
-                        font-size: 16px; /* Prevents zoom on iOS */
-                    }
-
-                    .result-item {
-                        padding: 1rem 0.75rem;
-                    }
-
-                    .hint-text {
-                        font-size: 0.8rem;
-                        padding: 0.5rem 1rem;
-                    }
-                }
-            </style>
-
-            <dialog aria-modal="true" aria-labelledby="${this.titleId}">
+    this.innerHTML = `
+      <datasette-modal><dialog aria-modal="true" aria-labelledby="${this.titleId}">
                 <div class="search-container">
                     <h2 id="${this.titleId}" class="visually-hidden">Jump to</h2>
                     <p id="${this.instructionsId}" class="visually-hidden">Type to search. Use up and down arrow keys to move through results, Enter to select a result, and Escape to close this menu.</p>
@@ -302,23 +47,22 @@ class NavigationSearch extends HTMLElement {
                         >
                         <button type="button" class="close-search" aria-label="Close jump menu">&times;</button>
                     </div>
-                    <div class="results-container"></div>
+                    <div class="modal-body results-container"></div>
                     <div class="hint-text">
                         <span><kbd>↑</kbd> <kbd>↓</kbd> Navigate</span>
                         <span><kbd>Enter</kbd> Select</span>
                         <span><kbd>Esc</kbd> Close</span>
                     </div>
                 </div>
-            </dialog>
+            </dialog></datasette-modal>
         `;
   }
 
   setupEventListeners() {
-    const dialog = this.shadowRoot.querySelector("dialog");
-    const input = this.shadowRoot.querySelector(".search-input");
-    const closeButton = this.shadowRoot.querySelector(".close-search");
-    const resultsContainer =
-      this.shadowRoot.querySelector(".results-container");
+    const dialog = this.querySelector("dialog");
+    const input = this.querySelector(".search-input");
+    const closeButton = this.querySelector(".close-search");
+    const resultsContainer = this.querySelector(".results-container");
 
     // Global keyboard listener for "/"
     document.addEventListener("keydown", (e) => {
@@ -355,8 +99,6 @@ class NavigationSearch extends HTMLElement {
       } else if (e.key === "Enter") {
         e.preventDefault();
         this.selectCurrentItem();
-      } else if (e.key === "Escape") {
-        this.closeMenu();
       }
     });
 
@@ -380,18 +122,6 @@ class NavigationSearch extends HTMLElement {
       }
     });
 
-    // Close on backdrop click
-    dialog.addEventListener("click", (e) => {
-      if (e.target === dialog) {
-        this.closeMenu();
-      }
-    });
-
-    dialog.addEventListener("cancel", (e) => {
-      e.preventDefault();
-      this.closeMenu();
-    });
-
     dialog.addEventListener("close", () => {
       this.onMenuClosed();
     });
@@ -432,19 +162,6 @@ class NavigationSearch extends HTMLElement {
     }
   }
 
-  focusRestoreTarget(trigger) {
-    if (trigger && typeof trigger.focus === "function") {
-      return trigger;
-    }
-    if (
-      document.activeElement &&
-      typeof document.activeElement.focus === "function"
-    ) {
-      return document.activeElement;
-    }
-    return null;
-  }
-
   setNavigationTriggersExpanded(expanded) {
     if (typeof document.querySelectorAll !== "function") {
       return;
@@ -465,8 +182,8 @@ class NavigationSearch extends HTMLElement {
   }
 
   updateComboboxState() {
-    const dialog = this.shadowRoot.querySelector("dialog");
-    const input = this.shadowRoot.querySelector(".search-input");
+    const dialog = this.querySelector("dialog");
+    const input = this.querySelector(".search-input");
     const matches = this.renderedMatches || [];
     this.setElementAttribute(
       input,
@@ -491,7 +208,7 @@ class NavigationSearch extends HTMLElement {
   }
 
   setStatus(message) {
-    const status = this.shadowRoot.querySelector(`#${this.statusId}`);
+    const status = this.querySelector(`#${this.statusId}`);
     if (status) {
       status.textContent = message || "";
     }
@@ -701,7 +418,7 @@ class NavigationSearch extends HTMLElement {
       section.render(node, {
         navigationSearch: this,
         container,
-        input: this.shadowRoot.querySelector(".search-input"),
+        input: this.querySelector(".search-input"),
       });
     });
   }
@@ -740,8 +457,8 @@ class NavigationSearch extends HTMLElement {
   }
 
   renderResults() {
-    const container = this.shadowRoot.querySelector(".results-container");
-    const input = this.shadowRoot.querySelector(".search-input");
+    const container = this.querySelector(".results-container");
+    const input = this.querySelector(".search-input");
     const showStartContent = !input.value.trim();
     const jumpSections = showStartContent ? this.jumpSections() : [];
     const startBlock = showStartContent
@@ -853,18 +570,15 @@ class NavigationSearch extends HTMLElement {
     }
   }
 
-  openMenu(trigger) {
-    const dialog = this.shadowRoot.querySelector("dialog");
-    const input = this.shadowRoot.querySelector(".search-input");
+  openMenu(returnFocusTo) {
+    const input = this.querySelector(".search-input");
 
-    this.restoreFocusTarget = this.focusRestoreTarget(trigger);
-    this.shouldRestoreFocus = true;
-    if (!dialog.open) {
-      dialog.showModal();
-    }
+    this.querySelector("datasette-modal").show({
+      returnFocusTo,
+      initialFocus: input,
+    });
     this.setNavigationTriggersExpanded(true);
     input.value = "";
-    input.focus();
 
     // Reset state, then populate the default jump list.
     this.matches = [];
@@ -874,29 +588,15 @@ class NavigationSearch extends HTMLElement {
   }
 
   closeMenu(options = {}) {
-    const dialog = this.shadowRoot.querySelector("dialog");
-    this.shouldRestoreFocus = options.restoreFocus !== false;
-    if (dialog.open) {
-      dialog.close();
-    } else {
-      this.onMenuClosed();
-    }
+    this.querySelector("datasette-modal").close(options);
   }
 
   onMenuClosed() {
-    const input = this.shadowRoot.querySelector(".search-input");
+    const input = this.querySelector(".search-input");
     this.setElementAttribute(input, "aria-expanded", "false");
     this.removeElementAttribute(input, "aria-activedescendant");
     this.setNavigationTriggersExpanded(false);
     this.setStatus("");
-    if (
-      this.shouldRestoreFocus &&
-      this.restoreFocusTarget &&
-      typeof this.restoreFocusTarget.focus === "function"
-    ) {
-      this.restoreFocusTarget.focus();
-    }
-    this.restoreFocusTarget = null;
   }
 
   escapeHtml(text) {

datasette/static/table.js

@@ -157,6 +157,7 @@ function createSetColumnTypeOption(value, name, description, checked) {
 
 function setSetColumnTypeDialogBusy(state, isBusy) {
   state.isBusy = isBusy;
+  state.modal.busy = isBusy;
   state.saveButton.disabled = isBusy;
   state.cancelButton.disabled = isBusy;
   Array.from(
@@ -185,7 +186,8 @@ function ensureSetColumnTypeDialog() {
     return null;
   }
 
-  var dialog = document.createElement("dialog");
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
   dialog.id = SET_COLUMN_TYPE_DIALOG_ID;
   dialog.className = "set-column-type-dialog";
   dialog.setAttribute("aria-labelledby", "set-column-type-title");
@@ -196,16 +198,17 @@ function ensureSetColumnTypeDialog() {
     </div>
     <p class="set-column-type-status"></p>
     <p class="set-column-type-error" hidden></p>
-    <div class="set-column-type-options"></div>
+    <div class="modal-body set-column-type-options"></div>
     <div class="modal-footer">
       <span class="footer-info"></span>
-      <button type="button" class="btn btn-ghost set-column-type-cancel">Cancel</button>
-      <button type="button" class="btn btn-primary set-column-type-save">Save</button>
+      <button type="button" class="modal-btn modal-btn-ghost set-column-type-cancel">Cancel</button>
+      <button type="button" class="modal-btn modal-btn-primary set-column-type-save">Save</button>
     </div>
   `;
-  document.body.appendChild(dialog);
+  document.body.appendChild(modal);
 
   setColumnTypeDialogState = {
+    modal: modal,
     dialog: dialog,
     meta: dialog.querySelector(".modal-meta"),
     status: dialog.querySelector(".set-column-type-status"),
@@ -220,21 +223,7 @@ function ensureSetColumnTypeDialog() {
   };
 
   setColumnTypeDialogState.cancelButton.addEventListener("click", function () {
-    if (!setColumnTypeDialogState.isBusy) {
-      dialog.close();
-    }
-  });
-
-  dialog.addEventListener("click", function (ev) {
-    if (ev.target === dialog && !setColumnTypeDialogState.isBusy) {
-      dialog.close();
-    }
-  });
-
-  dialog.addEventListener("cancel", function (ev) {
-    if (setColumnTypeDialogState.isBusy) {
-      ev.preventDefault();
-    }
+    modal.requestClose("cancel");
   });
 
   dialog.addEventListener("close", function () {
@@ -242,7 +231,9 @@ function ensureSetColumnTypeDialog() {
     setSetColumnTypeDialogBusy(setColumnTypeDialogState, false);
   });
 
-  setColumnTypeDialogState.saveButton.addEventListener("click", async function () {
+  setColumnTypeDialogState.saveButton.addEventListener(
+    "click",
+    async function () {
       var state = setColumnTypeDialogState;
       var selected = state.dialog.querySelector(
         'input[name="set-column-type-choice"]:checked',
@@ -253,7 +244,7 @@ function ensureSetColumnTypeDialog() {
         : "";
 
       if (selectedType === currentType) {
-      state.dialog.close();
+        state.modal.close();
         return;
       }
 
@@ -284,7 +275,8 @@ function ensureSetColumnTypeDialog() {
         setSetColumnTypeDialogBusy(state, false);
         showSetColumnTypeDialogError(state, error.message || "Request failed");
       }
-  });
+    },
+  );
 
   return setColumnTypeDialogState;
 }
@@ -341,9 +333,7 @@ function openSetColumnTypeDialog(th) {
     state.optionsWrap.appendChild(emptyState);
   }
 
-  if (!state.dialog.open) {
-    state.dialog.showModal();
-  }
+  state.modal.show();
   var selectedOption = state.dialog.querySelector(
     'input[name="set-column-type-choice"]:checked',
   );
@@ -367,9 +357,10 @@ function shouldShowShowAllColumns() {
 
 function hasMultipleVisibleColumns(manager) {
   return (
-    Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter(
-      (th) => th.dataset.column && th.dataset.isLinkColumn !== "1",
-    ).length > 1
+    Array.from(
+      document.querySelectorAll(manager.selectors.tableHeaders),
+    ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1")
+      .length > 1
   );
 }
 
@@ -649,10 +640,12 @@ function filterRowNumberFromName(name) {
 }
 
 function nextFilterRowNumber(manager) {
-  return filterRowsWithControls(manager).reduce((max, row) => {
+  return (
+    filterRowsWithControls(manager).reduce((max, row) => {
       var column = row.querySelector("select");
       return Math.max(max, filterRowNumberFromName(column && column.name));
-  }, 0) + 1;
+    }, 0) + 1
+  );
 }
 
 function setFilterRowNumber(row, number) {
@@ -679,9 +672,11 @@ function updateFilterRowButtons(manager) {
     if (addButton) {
       addButton.hidden = index !== rows.length - 1 || !column.value;
     }
-    var visibleButtonCount = [removeButton, addButton].filter(function (button) {
+    var visibleButtonCount = [removeButton, addButton].filter(
+      function (button) {
         return button && !button.hidden;
-    }).length;
+      },
+    ).length;
     row.classList.toggle(
       "filter-controls-row-has-buttons",
       visibleButtonCount > 0,
@@ -703,7 +698,9 @@ function cloneFilterRow(row) {
   clone.querySelector(".filter-op select").name = "_filter_op";
   clone.querySelector("input.filter-value").name = "_filter_value";
   resetFilterRow(clone);
-  clone.querySelectorAll(".filter-row-icon").forEach((button) => button.remove());
+  clone
+    .querySelectorAll(".filter-row-icon")
+    .forEach((button) => button.remove());
   return clone;
 }
 
@@ -860,10 +857,45 @@ function openColumnChooser() {
   });
 }
 
+function initCountAll() {
+  var button = document.querySelector(".count-all");
+  if (!button) {
+    return;
+  }
+  button.addEventListener("click", async function () {
+    var count = document.querySelector(".table-count");
+    var error = document.querySelector(".count-error");
+    button.disabled = true;
+    button.textContent = "Counting…";
+    error.textContent = "";
+    try {
+      var response = await fetch(button.dataset.countUrl + location.search, {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+        },
+      });
+      var data = await response.json();
+      if (!response.ok || !data.ok) {
+        throw new Error((data.errors || ["Count failed"]).join(" "));
+      }
+      count.textContent =
+        data.count.toLocaleString("en-US") +
+        (data.count === 1 ? " row" : " rows");
+      button.remove();
+    } catch (ex) {
+      error.textContent = ex.message || "Count failed";
+      button.disabled = false;
+      button.textContent = "count all";
+    }
+  });
+}
+
 // Ensures Table UI is initialized only after the Manager is ready.
 document.addEventListener("datasette_init", function (evt) {
   const { detail: manager } = evt;
 
+  initCountAll();
   initializeColumnActions(manager);
 
   // Main table

datasette/templates/api_explorer.html

@@ -3,7 +3,6 @@
 {% block title %}API Explorer{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% endblock %}
 
 {% block content %}
@@ -126,7 +125,7 @@ getForm.addEventListener("submit", (ev) => {
     document.getElementById('response-status').textContent = response.status;
     return response.json();
   }).then((data) => {
-    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
+    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
     errorList.style.display = 'none';
   }).catch((error) => {
     alert(error);
@@ -174,7 +173,7 @@ postForm.addEventListener("submit", (ev) => {
     } else {
       errorList.style.display = 'none';
     }
-    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
+    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
     output.style.display = 'block';
   }).catch(err => {
     alert("Error: " + err);

datasette/templates/base.html

@@ -8,6 +8,7 @@
     <link rel="stylesheet" href="{{ url.url }}"{% if url.get("sri") %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}>
 {% endfor %}
 <script>window.datasetteVersion = '{{ datasette_version }}';</script>
+<script src="{{ static('modal.js') }}" defer></script>
 <script src="{{ static('datasette-manager.js') }}" defer></script>
 {% for url in extra_js_urls %}
     <script {% if url.module %}type="module" {% endif %}src="{{ url.url }}"{% if url.get("sri") %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}></script>

datasette/templates/debug_allowed.html

@@ -3,7 +3,6 @@
 {% block title %}Allowed Resources{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -198,7 +197,7 @@ function displayResults(data) {
     }
 
     // Update raw JSON
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayError(data) {
@@ -208,7 +207,7 @@ function displayError(data) {
 
     resultsContent.innerHTML = `<div class="error-message">Error: ${escapeHtml(data.error || 'Unknown error')}</div>`;
 
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 // Disable child input if parent is empty

datasette/templates/debug_check.html

@@ -3,7 +3,6 @@
 {% block title %}Explain a permission decision{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 <style>
@@ -238,7 +237,7 @@ function displayResult(data) {
     displayRules(data.explanation);
     displayRestrictions(data.explanation.restrictions);
     displayRequirements(data.explanation.required_actions);
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayRules(explanation) {
@@ -298,7 +297,7 @@ function displayError(data) {
     document.getElementById('matching-rules').innerHTML = '';
     document.getElementById('restrictions-section').style.display = 'none';
     document.getElementById('requirements-section').style.display = 'none';
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 form.addEventListener('submit', event => {

datasette/templates/debug_rules.html

@@ -3,7 +3,6 @@
 {% block title %}Permission Rules{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -185,7 +184,7 @@ function displayResults(data) {
     }
 
     // Update raw JSON
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayError(data) {
@@ -195,7 +194,7 @@ function displayError(data) {
 
     resultsContent.innerHTML = `<div class="error-message">Error: ${escapeHtml(data.error || 'Unknown error')}</div>`;
 
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 </script>

datasette/templates/table.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 
-{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
+{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
 
 {% block extra_head %}
 {{- super() -}}
@@ -47,11 +47,12 @@
 {% endif %}
 
 {% if count or human_description_en %}
-    <h3>
-        {% if count_truncated %}&gt;{{ "{:,}".format(count - 1) }} rows
-        {% if allow_execute_sql and query.sql %} <a class="count-sql" style="font-size: 0.8em;" href="{{ urls.database_query(database, count_sql) }}">count all</a>{% endif %}
+    <h3 class="table-summary">
+        {% if count_truncated %}<span class="table-count" aria-live="polite">{{ "{:,}".format(count - 1) }}+ rows</span>
+        <button type="button" class="count-all" data-count-url="{{ urls.table(database, table) }}/-/count">count all</button>
+        <span class="count-error" role="alert"></span>
         {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}
-        {% if human_description_en %}{{ human_description_en }}{% endif %}
+        {% if human_description_en %}<span class="table-summary-description">{{ human_description_en }}</span>{% endif %}
     </h3>
 {% endif %}
 

datasette/utils/__init__.py

@@ -568,7 +568,7 @@ def escape_css_string(s):
 
 
 def escape_sqlite(s):
-    if _boring_keyword_re.match(s) and (s.lower() not in reserved_words):
+    if _boring_keyword_re.fullmatch(s) and (s.lower() not in reserved_words):
         return s
     return '"{}"'.format(s.replace('"', '""'))
 
@@ -820,7 +820,8 @@ def detect_spatialite(conn):
 
 def detect_fts(conn, table):
     """Detect if table has a corresponding FTS virtual table and return it"""
-    rows = conn.execute(detect_fts_sql(table)).fetchall()
+    sql, params = detect_fts_sql(table)
+    rows = conn.execute(sql, params).fetchall()
     if len(rows) == 0:
         return None
     else:
@@ -828,18 +829,26 @@ def detect_fts(conn, table):
 
 
 def detect_fts_sql(table):
-    return r"""
+    escaped_table = table.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+    return (
+        r"""
             select name from sqlite_master
                 where rootpage = 0
                 and (
-                sql like '%VIRTUAL TABLE%USING FTS%content="{table}"%'
-                or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%'
+                    sql like :fts_double_quoted escape char(92)
+                    or sql like :fts_bracket_quoted escape char(92)
                     or (
-                    tbl_name = "{table}"
+                        tbl_name = :table
                         and sql like '%VIRTUAL TABLE%USING FTS%'
                     )
                 )
-    """.format(table=table.replace("'", "''"))
+        """,
+        {
+            "fts_double_quoted": f'%VIRTUAL TABLE%USING FTS%content="{escaped_table}"%',
+            "fts_bracket_quoted": f"%VIRTUAL TABLE%USING FTS%content=[{escaped_table}]%",
+            "table": table,
+        },
+    )
 
 
 def detect_json1(conn=None):
@@ -1557,7 +1566,13 @@ async def row_sql_params_pks(db, table, pk_values):
     if use_rowid:
         select = "rowid, *"
         pks = ["rowid"]
-    wheres = [f'"{pk}"=:p{i}' for i, pk in enumerate(pks)]
+    wheres = []
+    for i, pk in enumerate(pks):
+        escaped_pk = escape_sqlite(pk)
+        # Preserve the historic always-quoted SQL exposed by _extra=query
+        if escaped_pk == pk:
+            escaped_pk = f'"{pk}"'
+        wheres.append(f"{escaped_pk}=:p{i}")
     sql = f"select {select} from {escape_sqlite(table)} where {' AND '.join(wheres)}"
     params = {}
     for i, pk_value in enumerate(pk_values):
@@ -1729,7 +1744,7 @@ def redact_keys(original: dict, key_patterns: Iterable) -> dict:
             return {
                 k: (
                     redact(v)
-                    if not any(pattern in k for pattern in key_patterns)
+                    if not any(pattern in k.casefold() for pattern in key_patterns)
                     else "***"
                 )
                 for k, v in data.items()

datasette/utils/actions_sql.py

@@ -29,6 +29,15 @@ from datasette.utils.permissions import gather_permission_sql_from_hooks
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
+    from datasette.permissions import Action
+
+
+def _child_collation(action: "Action") -> str:
+    """Match resource identity without changing the spelling returned by SQL."""
+    resource_class = action.resource_class
+    if resource_class is not None and resource_class.case_insensitive_child:
+        return "NOCASE"
+    return "BINARY"
 
 
 async def build_allowed_resources_sql(
@@ -149,6 +158,7 @@ async def _build_single_action_sql(
         raise ValueError(f"Unknown action: {action}")
 
     # Get base resources SQL from the resource class
+    child_collation = _child_collation(action_obj)
     base_resources_sql = await action_obj.resource_class.resources_sql(
         datasette, actor=actor
     )
@@ -185,7 +195,7 @@ async def _build_single_action_sql(
         if permission_sql.sql is None:
             continue
         rule_sqls.append(f"""
-            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
+            SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
             """.strip())
@@ -299,9 +309,9 @@ async def _build_single_action_sql(
         query_parts.extend(
             ["anon_child_agg AS ("]
             + _anon_agg(
-                "parent, child,",
+                f"parent, child COLLATE {child_collation} AS child,",
                 "parent IS NOT NULL AND child IS NOT NULL",
-                "parent, child",
+                f"parent, child COLLATE {child_collation}",
             )
             + ["),", "anon_parent_agg AS ("]
             + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent")
@@ -382,7 +392,8 @@ async def _build_single_action_sql(
         # Wrap each restriction_sql in a subquery to avoid operator precedence issues
         # with UNION ALL inside the restriction SQL statements
         restriction_intersect = "\nINTERSECT\n".join(
-            f"SELECT * FROM ({sql})" for sql in restriction_sqls
+            f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})"
+            for sql in restriction_sqls
         )
         # Decompose by NULL-pattern so the final filter can use pure-equality
         # EXISTS lookups (satisfiable via automatic indexes) instead of a
@@ -480,6 +491,7 @@ async def build_permission_rules_sql(
     union_parts = []
     all_params = {}
     restriction_sqls = []
+    child_collation = _child_collation(action_obj)
 
     for permission_sql in permission_sqls:
         all_params.update(permission_sql.params or {})
@@ -493,7 +505,7 @@ async def build_permission_rules_sql(
             continue
 
         union_parts.append(f"""
-            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
+            SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
             """.strip())
@@ -564,6 +576,7 @@ async def check_permissions_for_actions(
     verdicts = {}
 
     for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)):
+        child_collation = _child_collation(datasette.actions[action])
         prefix = f"a{i}_"
         rule_parts = []
         restriction_parts = []
@@ -589,7 +602,7 @@ async def check_permissions_for_actions(
             if sql is None:
                 continue
             rule_parts.append(
-                f"SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
+                f"SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
             )
 
         if not rule_parts:
@@ -623,7 +636,8 @@ async def check_permissions_for_actions(
         if restriction_parts:
             # Database-level restrictions (parent, NULL) match all children
             restriction_intersect = "\nINTERSECT\n".join(
-                f"SELECT * FROM ({sql})" for sql in restriction_parts
+                f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})"
+                for sql in restriction_parts
             )
             ctes.append(f"a{i}_restriction AS (\n{restriction_intersect}\n)")
             verdict_sql = f"""({verdict_sql}) AND EXISTS (
@@ -770,6 +784,7 @@ async def _explain_single_action(
     db = datasette.get_internal_database()
     matched_rules = []
     restrictions = []
+    child_collation = _child_collation(datasette.actions[action])
 
     for permission_sql in permission_sqls:
         params = dict(permission_sql.params or {})
@@ -784,7 +799,7 @@ async def _explain_single_action(
                 SELECT parent, child, allow, reason
                 FROM ({permission_sql.sql}) AS permission_rules
                 WHERE (parent IS NULL OR parent = :{parent_param})
-                  AND (child IS NULL OR child = :{child_param})
+                  AND (child IS NULL OR child COLLATE {child_collation} = :{child_param})
                 """,
                 params,
             )
@@ -811,7 +826,7 @@ async def _explain_single_action(
                     SELECT EXISTS(
                         SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules
                         WHERE (parent IS NULL OR parent = :{parent_param})
-                          AND (child IS NULL OR child = :{child_param})
+                          AND (child IS NULL OR child COLLATE {child_collation} = :{child_param})
                     ) AS resource_is_in_allowlist
                     """,
                     params,

datasette/utils/asgi.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import re
 from http.cookies import Morsel, SimpleCookie
@@ -82,6 +83,19 @@ SAMESITE_VALUES = ("strict", "lax", "none")
 DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024  # 2MB
 
 
+class _RequestHeaders(dict):
+    """Incoming headers with lowercase keys and case-insensitive lookups."""
+
+    def __getitem__(self, key):
+        return super().__getitem__(key.lower())
+
+    def get(self, key, default=None):
+        return super().get(key.lower(), default)
+
+    def __contains__(self, key):
+        return super().__contains__(key.lower())
+
+
 class Request:
     def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES):
         self.scope = scope
@@ -111,10 +125,10 @@ class Request:
 
     @property
     def headers(self):
-        return {
-            k.decode("latin-1").lower(): v.decode("latin-1")
+        return _RequestHeaders(
+            (k.decode("latin-1").lower(), v.decode("latin-1"))
             for k, v in self.scope.get("headers") or []
-        }
+        )
 
     @property
     def host(self):
@@ -300,12 +314,24 @@ class AsgiLifespan:
             while True:
                 message = await receive()
                 if message["type"] == "lifespan.startup":
+                    try:
                         for fn in self.on_startup:
                             await fn()
+                    except Exception as e:  # noqa: BLE001
+                        await send(
+                            {"type": "lifespan.startup.failed", "message": str(e)}
+                        )
+                        return
                     await send({"type": "lifespan.startup.complete"})
                 elif message["type"] == "lifespan.shutdown":
+                    try:
                         for fn in self.on_shutdown:
                             await fn()
+                    except Exception as e:  # noqa: BLE001
+                        await send(
+                            {"type": "lifespan.shutdown.failed", "message": str(e)}
+                        )
+                        return
                     await send({"type": "lifespan.shutdown.complete"})
                     return
         else:
@@ -485,6 +511,8 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None):
             await asgi_send_html(send, "404: File not found", 404)
             return
 
+    # Only the actual static-file handler can bypass dynamic response privacy.
+    inner_static._datasette_static = True
     return inner_static
 
 
@@ -624,10 +652,23 @@ class AsgiRunOnFirstRequest:
         self.asgi = asgi
         self.on_startup = on_startup
         self._started = False
+        # Guards against concurrent early requests interleaving with startup:
+        # without this, several requests could all observe `_started is
+        # False` and proceed before any of them finish running the hooks.
+        self._lock = asyncio.Lock()
 
     async def __call__(self, scope, receive, send):
+        # Leave "lifespan" scope events alone - this shim only exists as a
+        # fallback for hosts that never send them. It wraps AsgiLifespan, so
+        # if it ran on_startup here too, a startup exception would escape
+        # before AsgiLifespan's own try/except got a chance to turn it into
+        # a lifespan.startup.failed message.
+        if scope["type"] != "lifespan" and not self._started:
+            async with self._lock:
+                # Re-check: another request may have finished startup while
+                # we were waiting for the lock.
                 if not self._started:
-            self._started = True
                     for hook in self.on_startup:
                         await hook()
+                    self._started = True
         return await self.asgi(scope, receive, send)

datasette/utils/multipart.py

@@ -358,6 +358,13 @@ class MultipartParser:
         self.buffer.extend(chunk)
         self._process()
 
+    def close(self) -> None:
+        """Discard completed uploads and any file still being received."""
+        if self.current_file is not None:
+            self.current_file.close()
+            self.current_file = None
+        self.form_data.close()
+
     def _process(self) -> None:
         """Process buffered data."""
         while True:
@@ -577,6 +584,9 @@ class MultipartParser:
     def _finish_part(self) -> None:
         """Finalize current part and add to form data."""
         if self.current_name is None:
+            if self.current_file is not None:
+                self.current_file.close()
+                self.current_file = None
             return
 
         if self.current_filename is not None:
@@ -722,12 +732,29 @@ async def parse_form_data(
         batch_target = 64 * 1024
         batch = bytearray()
 
+        async def run_parser(fn, *args):
+            # Cancellation must not close files while a worker is using them.
+            task = asyncio.create_task(asyncio.to_thread(fn, *args))
+            try:
+                return await asyncio.shield(task)
+            except asyncio.CancelledError as cancelled:
+                try:
+                    while not task.done():
+                        try:
+                            await asyncio.shield(task)
+                        except asyncio.CancelledError:
+                            continue
+                    task.result()
+                finally:
+                    raise cancelled
+
         async def flush_batch() -> None:
             if batch:
                 data = bytes(batch)
                 batch.clear()
-                await asyncio.to_thread(parser.feed, data)
+                await run_parser(parser.feed, data)
 
+        try:
             while True:
                 message = await receive()
                 message_type = message.get("type")
@@ -744,7 +771,11 @@ async def parse_form_data(
                     break
 
             await flush_batch()
-        return await asyncio.to_thread(parser.finalize)
+            return await run_parser(parser.finalize)
+        except BaseException:
+            # No FormData is returned to the caller to take ownership on failure.
+            await asyncio.to_thread(parser.close)
+            raise
 
     else:
         raise MultipartParseError(

datasette/utils/sql_analysis.py

@@ -1,6 +1,8 @@
+import sys
 from dataclasses import dataclass
 from typing import Literal
 
+from datasette.utils import escape_sqlite
 from datasette.utils.sqlite import SQLiteTableType, sqlite3, sqlite_table_type
 
 SQLOperation = Literal[
@@ -195,6 +197,16 @@ def _allow_authorizer_action(*args):
     return sqlite3.SQLITE_OK
 
 
+def _disable_authorizer(conn):
+    # Python 3.11 added support for unregistering an authorizer using None.
+    # On Python 3.10, None is installed as the callback instead, and the next
+    # statement fails with "not authorized" when sqlite3 tries to call it.
+    if sys.version_info >= (3, 11):
+        conn.set_authorizer(None)
+    else:
+        conn.set_authorizer(_allow_authorizer_action)
+
+
 def analyze_sql_tables(
     conn,
     sql: str,
@@ -208,7 +220,9 @@ def analyze_sql_tables(
 
     This function is synchronous and connection-based. It temporarily installs a
     SQLite authorizer, prepares ``EXPLAIN <sql>``, and returns the operation
-    callbacks observed while SQLite compiles the statement.
+    callbacks observed while SQLite compiles the statement. ``CREATE VIEW`` is
+    additionally executed inside a rolled-back savepoint so its source-table reads
+    can be discovered by analyzing a query against the temporary view.
     """
     operations: dict[OperationKey, set[str]] = {}
 
@@ -481,7 +495,7 @@ def analyze_sql_tables(
                         conn, key.table, schema=key.sqlite_schema
                     )
     finally:
-        conn.set_authorizer(None)
+        _disable_authorizer(conn)
 
     has_schema_operation = any(
         key.target_type in {"table", "index", "view", "trigger", "virtual-table"}
@@ -532,7 +546,7 @@ def analyze_sql_tables(
             return None
         return table_kind_cache[(key.sqlite_schema, key.table)]
 
-    return SQLAnalysis(
+    analysis = SQLAnalysis(
         operations=tuple(
             Operation(
                 operation=key.operation,
@@ -549,3 +563,58 @@ def analyze_sql_tables(
             for key, columns in operations.items()
         )
     )
+
+    # SQLite does not resolve the SELECT body of a view when preparing CREATE
+    # VIEW, so its authorizer does not report reads from the view's source
+    # tables. Temporarily create the view, analyze a query against it (which
+    # does resolve the body), then roll the schema change back. Database-level
+    # callers use an isolated writable connection for this analysis.
+    create_view_operations = tuple(
+        operation
+        for operation in analysis.operations
+        if operation.operation == "create" and operation.target_type == "view"
+    )
+    if not create_view_operations:
+        return analysis
+
+    savepoint = "datasette_analyze_create_view"
+    conn.execute(f"SAVEPOINT {savepoint}")
+    try:
+        conn.execute(sql, params if params is not None else {})
+        dependency_reads = []
+        for view_operation in create_view_operations:
+            if view_operation.sqlite_schema is None or view_operation.table is None:
+                raise sqlite3.OperationalError(
+                    "Could not determine the created view name"
+                )
+            quoted_schema = escape_sqlite(view_operation.sqlite_schema)
+            quoted_view = escape_sqlite(view_operation.table)
+            qualified_view = f"{quoted_schema}.{quoted_view}"
+            view_analysis = analyze_sql_tables(
+                conn,
+                f"SELECT * FROM {qualified_view}",
+                database_name=database_name,
+                schema_to_database=schema_to_database,
+            )
+            dependency_reads.extend(
+                operation
+                for operation in view_analysis.operations
+                if operation.operation == "read"
+                and not (
+                    operation.sqlite_schema == view_operation.sqlite_schema
+                    and operation.table == view_operation.table
+                )
+            )
+    finally:
+        conn.execute(f"ROLLBACK TO {savepoint}")
+        conn.execute(f"RELEASE {savepoint}")
+
+    existing_operations = set(analysis.operations)
+    return SQLAnalysis(
+        operations=analysis.operations
+        + tuple(
+            operation
+            for operation in dependency_reads
+            if operation not in existing_operations
+        )
+    )

datasette/utils/sqlite.py

@@ -15,8 +15,17 @@ if hasattr(sqlite3, "enable_callback_tracebacks"):
 _cached_sqlite_version = None
 _cached_supports_returning = None
 SQLiteTableType = Literal["table", "view", "virtual", "shadow"]
+_SQLITE_IDENTIFIER_RE = (
+    r"""(?:"(?:[^"]|"")*"|'(?:[^']|'')*'|`(?:[^`]|``)*`|\[[^\]]*\]|[^\s.()'"`\[\]]+)"""
+)
 _VIRTUAL_TABLE_MODULE_RE = re.compile(
-    r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)",
+    r"^\s*CREATE\s+VIRTUAL\s+TABLE\b\s*(?:IF\s+NOT\s+EXISTS\s+)?"
+    + _SQLITE_IDENTIFIER_RE
+    + r"(?:\s*\.\s*"
+    + _SQLITE_IDENTIFIER_RE
+    + r")?\s*\bUSING\b\s*("
+    + _SQLITE_IDENTIFIER_RE
+    + r")",
     re.IGNORECASE | re.DOTALL,
 )
 _VIRTUAL_TABLE_SHADOW_SUFFIXES = {
@@ -83,19 +92,53 @@ def sqlite_table_type(
 ) -> SQLiteTableType | None:
     if supports_table_list():
         try:
-            query = "select type from pragma_table_list where name = ?"
-            params: tuple[str, ...] = (table,)
+            # Use the "PRAGMA table_list" statement form rather than the
+            # pragma_table_list(...) table-valued function. The
+            # table-valued function is resolved like an ordinary relation
+            # name, so an attacker-created table or view literally named
+            # "pragma_table_list" can shadow it and spoof the reported
+            # type (e.g. claiming a virtual table is an ordinary table).
+            # The PRAGMA statement form is a distinct piece of SQL syntax
+            # that always invokes SQLite's built-in pragma, so it cannot
+            # be shadowed by a user-created relation.
             if schema is not None:
-                query += " and schema = ?"
-                params = (table, schema)
-            row = conn.execute(query, params).fetchone()
-            if row is not None and row[0] in {"table", "view", "virtual", "shadow"}:
-                return row[0]
+                query = f"PRAGMA {_quote_identifier(schema)}.table_list"
+            else:
+                query = "PRAGMA table_list"
+            cursor = conn.execute(query)
+            columns = [description[0] for description in cursor.description]
+            for row in cursor.fetchall():
+                record = dict(zip(columns, row))
+                if record.get("name") != table:
+                    continue
+                if schema is not None and record.get("schema") != schema:
+                    continue
+                row_type = record.get("type")
+                if row_type in {"table", "view", "virtual", "shadow"}:
+                    return row_type
         except sqlite3.DatabaseError:
             pass
     return _sqlite_table_type_from_schema(conn, table, schema=schema)
 
 
+def check_structured_write_table(conn, table: str, *, allow_missing=False):
+    """Validate a row-write target on the connection that will perform the write."""
+    # SQLite resolves identifiers case-insensitively. The create API must not
+    # treat a differently cased existing name as a missing table.
+    row = conn.execute(
+        "select name from main.sqlite_master where name = ? collate nocase "
+        "and type in ('table', 'view')",
+        (table,),
+    ).fetchone()
+    if row is None and allow_missing:
+        return
+    if row is not None and sqlite_table_type(conn, row[0]) == "table":
+        return
+    # Virtual table modules can interpret row writes as administrative operations.
+    # Their shadow tables are internal storage, not independently writable data.
+    raise ValueError("Structured writes require an ordinary table")
+
+
 def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]:
     schema_table = _sqlite_schema_table(schema)
     try:
@@ -118,6 +161,63 @@ def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]
     return sorted(hidden_tables) + content_fts_tables
 
 
+def sqlite_derived_table_dependencies(
+    conn, *, schema: str | None = "main"
+) -> dict[str, str]:
+    """Return implementation table -> logical/content table dependencies.
+
+    ``PRAGMA table_list`` safely identifies virtual and shadow tables, but
+    does not report which virtual table owns a shadow table or which table is
+    named by an FTS ``content=`` option. Derive those relationships from
+    ``sqlite_master`` DDL and the documented shadow-table suffixes.
+
+    Database errors propagate: failed discovery must not be mistaken for an
+    empty dependency map and cached as permission to skip inheritance.
+    """
+    schema_table = _sqlite_schema_table(schema)
+    rows = conn.execute(
+        f"select name, sql from {schema_table} where type = 'table'"
+    ).fetchall()
+
+    table_names = {row[0] for row in rows}
+    # SQLite identifiers fold ASCII letters only.
+    identifier_case = str.maketrans(
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
+    )
+    canonical_names = {name.translate(identifier_case): name for name in table_names}
+    dependencies = {}
+    for virtual_table, sql in rows:
+        module = _virtual_table_module(sql)
+        if module is None:
+            continue
+
+        # SQLite's documented shadow tables are implementation details of
+        # their logical virtual table.
+        for suffix in _VIRTUAL_TABLE_SHADOW_SUFFIXES.get(module, ()):
+            shadow_table = virtual_table + suffix
+            if shadow_table in table_names:
+                dependencies[shadow_table] = virtual_table
+
+        # An external-content FTS table can expose values fetched from its
+        # content table, so it must also depend on that table's permission.
+        if module in {"fts3", "fts4", "fts5"}:
+            content_table = _fts_external_content_table(sql)
+            if content_table:
+                dependencies[virtual_table] = content_table
+
+        if module in {"fts5vocab", "fts4aux"}:
+            source = _fts_vocabulary_source(sql, module, schema or "main")
+            source = (
+                canonical_names.get(source.translate(identifier_case))
+                if source
+                else None
+            )
+            # An unresolved source is itself derived, so the one-hop policy denies it.
+            dependencies[virtual_table] = source or virtual_table
+
+    return dependencies
+
+
 def _sqlite_table_type_from_schema(
     conn,
     table: str,
@@ -184,10 +284,151 @@ def _quote_identifier(value: str) -> str:
 def _virtual_table_module(sql: str | None) -> str | None:
     if not sql:
         return None
+    match = _VIRTUAL_TABLE_MODULE_RE.search(_strip_sql_comments(sql))
+    if match is None:
+        return None
+    return _unquote_sql_value(match.group(1)).lower()
+
+
+def _fts_external_content_table(sql: str | None) -> str | None:
+    """Extract the external ``content=`` table from an FTS declaration."""
+    if not sql:
+        return None
+    sql = _strip_sql_comments(sql)
     match = _VIRTUAL_TABLE_MODULE_RE.search(sql)
     if match is None:
         return None
-    return match.group(1).strip("\"'[]`").lower()
+    open_paren = sql.find("(", match.end())
+    if open_paren == -1:
+        return None
+    close_paren = sql.rfind(")")
+    if close_paren <= open_paren:
+        return None
+
+    for argument in _split_sql_arguments(sql[open_paren + 1 : close_paren]):
+        key, separator, value = argument.partition("=")
+        if not separator or key.strip().lower() != "content":
+            continue
+        return _unquote_sql_value(value.strip())
+    return None
+
+
+def _fts_vocabulary_source(sql: str, module: str, schema: str) -> str | None:
+    """Resolve a vocabulary source within the current SQLite schema.
+
+    Cross-schema sources cannot be represented by the dependency map and
+    are conservatively left unresolved.
+    """
+    sql = _strip_sql_comments(sql)
+    match = _VIRTUAL_TABLE_MODULE_RE.search(sql)
+    if match is None:
+        return None
+    start = sql.find("(", match.end())
+    end = sql.rfind(")")
+    if start < 0 or end <= start:
+        return None
+    arguments = [
+        _unquote_sql_value(arg.strip())
+        for arg in _split_sql_arguments(sql[start + 1 : end])
+    ]
+    expected = 2 if module == "fts5vocab" else 1
+    if len(arguments) == expected:
+        return arguments[0]
+    if len(arguments) == expected + 1 and arguments[0].lower() == schema.lower():
+        return arguments[1]
+    return None
+
+
+def _split_sql_arguments(arguments: str) -> list[str]:
+    """Split comma-separated SQLite arguments without splitting quoted text."""
+    parts = []
+    start = 0
+    quote = None
+    closing_quote = None
+    index = 0
+    while index < len(arguments):
+        char = arguments[index]
+        if quote is None:
+            if char in {"'", '"', "`", "["}:
+                quote = char
+                closing_quote = "]" if char == "[" else char
+            elif char == ",":
+                parts.append(arguments[start:index])
+                start = index + 1
+        elif char == closing_quote:
+            # Single/double/backtick quoting escapes the delimiter by
+            # doubling it. Square-bracket identifiers do not.
+            if (
+                quote != "["
+                and index + 1 < len(arguments)
+                and arguments[index + 1] == closing_quote
+            ):
+                index += 1
+            else:
+                quote = None
+                closing_quote = None
+        index += 1
+    parts.append(arguments[start:])
+    return parts
+
+
+def _strip_sql_comments(sql: str) -> str:
+    """Remove SQLite comments while preserving quoted strings/identifiers."""
+    output = []
+    quote = None
+    closing_quote = None
+    index = 0
+    while index < len(sql):
+        char = sql[index]
+        next_char = sql[index + 1] if index + 1 < len(sql) else ""
+        if quote is None:
+            if char in {"'", '"', "`", "["}:
+                quote = char
+                closing_quote = "]" if char == "[" else char
+                output.append(char)
+            elif char == "-" and next_char == "-":
+                index += 2
+                while index < len(sql) and sql[index] not in "\r\n":
+                    index += 1
+                output.append(" ")
+                continue
+            elif char == "/" and next_char == "*":
+                index += 2
+                while index + 1 < len(sql) and sql[index : index + 2] != "*/":
+                    index += 1
+                index = min(index + 2, len(sql))
+                output.append(" ")
+                continue
+            else:
+                output.append(char)
+        else:
+            output.append(char)
+            if char == closing_quote:
+                if (
+                    quote != "["
+                    and index + 1 < len(sql)
+                    and sql[index + 1] == closing_quote
+                ):
+                    output.append(sql[index + 1])
+                    index += 1
+                else:
+                    quote = None
+                    closing_quote = None
+        index += 1
+    return "".join(output)
+
+
+def _unquote_sql_value(value: str) -> str:
+    if len(value) < 2:
+        return value
+    pairs = {"'": "'", '"': '"', "`": "`", "[": "]"}
+    closing = pairs.get(value[0])
+    if closing is None or value[-1] != closing:
+        return value
+    unquoted = value[1:-1]
+    if value[0] != "[":
+        unquoted = unquoted.replace(closing * 2, closing)
+    return unquoted
 
 
 def _is_fts_content_virtual_table(sql: str | None) -> bool:

datasette/utils/testing.py

@@ -4,7 +4,7 @@ from urllib.parse import urlencode
 from asgiref.sync import async_to_sync
 
 # These wrapper classes pre-date the introduction of
-# datasette.client and httpx to Datasette. They could
+# datasette.client and httpx2 to Datasette. They could
 # be removed if the Datasette tests are modified to
 # call datasette.client directly.
 

datasette/version.py

@@ -1,2 +1,2 @@
-__version__ = "1.0a38"
+__version__ = "1.0a40"
 __version_info__ = tuple(__version__.split("."))

datasette/views/base.py

@@ -35,7 +35,7 @@ class DatasetteError(Exception):
         self.error_dict = error_dict or {}
         self.status = status
         self.message_is_html = message_is_html
-        # Plain text used for JSON error responses when message is HTML
+        # Plain text used for JSON and CSV error responses when message is HTML
         self.plain_message = plain_message
 
 

datasette/views/database.py

@@ -40,7 +40,11 @@ from datasette.write_sql import QueryWriteRejected
 
 from . import Context
 from .base import DatasetteError, View, stream_csv
-from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns
+from .query_helpers import (
+    _block_framing,
+    _ensure_stored_query_execution_permissions,
+    _table_columns,
+)
 from .table_create_alter import _create_table_ui_context
 from .table_extras import (
     QueryExtraContext,
@@ -857,7 +861,8 @@ class QueryView(View):
                 raise DatasetteError("?sql= is required", status=400)
 
             async def fetch_data_for_csv(request, _next=None):
-                results = await db.execute(sql, params, truncate=True)
+                # Reuse the trusted magic parameter values prepared above.
+                results = await db.execute(sql, params_for_query, truncate=True)
                 data = {"rows": results.rows, "columns": results.columns}
                 return data, None, None
 
@@ -1140,6 +1145,8 @@ class QueryView(View):
             assert False, f"Invalid format: {format_}"
         if datasette.cors:
             add_cors_headers(r.headers)
+        if stored_query_write and format_ == "html":
+            _block_framing(r)
         return r
 
 

datasette/views/execute_write.py

@@ -1,6 +1,7 @@
 import re
 from urllib.parse import urlencode
 
+from datasette.database import QueryInterrupted
 from datasette.resources import DatabaseResource
 from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3
 from datasette.utils.asgi import Response
@@ -384,7 +385,7 @@ class ExecuteWriteView(BaseView):
         try:
             execute_write_kwargs = {"request": request}
             cursor = await db.execute_write(sql, params, **execute_write_kwargs)
-        except sqlite3.DatabaseError as ex:
+        except (QueryInterrupted, sqlite3.DatabaseError) as ex:
             message = str(ex)
             if wants_json:
                 return _block_framing(Response.error([message], 400))

datasette/views/row.py

@@ -28,9 +28,11 @@ from datasette.utils import (
     path_with_format,
     path_with_removed_args,
     sqlite3,
+    tilde_decode,
     to_css_class,
 )
 from datasette.utils.asgi import Forbidden, NotFound, PayloadTooLarge, Response
+from datasette.utils.sqlite import check_structured_write_table
 
 from . import Context, from_extra
 from .base import BaseView, DatasetteError, stream_csv
@@ -137,6 +139,12 @@ class RowContext(Context):
     )
 
 
+async def _database_and_table_resource_from_request(datasette, request):
+    db = await datasette.resolve_database(request)
+    table = tilde_decode(request.url_vars["table"])
+    return db, table, TableResource(database=db.name, table=table)
+
+
 class RowView(BaseView):
     name = "row"
 
@@ -263,7 +271,7 @@ class RowView(BaseView):
         if ttl is None or not ttl.isdigit():
             ttl = self.ds.setting("default_cache_ttl")
 
-        return self.set_response_headers(response, ttl)
+        return self.set_response_headers(response, ttl, request)
 
     async def html(self, request, data, extra_template_data, templates):
         extras = {}
@@ -376,9 +384,17 @@ class RowView(BaseView):
             },
         )
 
-    def set_response_headers(self, response, ttl):
+    def set_response_headers(self, response, ttl, request=None):
+        private = getattr(request, "_datasette_private_response", False)
         # Set far-future cache expiry
         if self.ds.cache_headers and response.status == 200:
+            if private:
+                # This response is only visible to the current actor (denied
+                # to anonymous requests), so it must never be stored by a
+                # shared cache/CDN - and ?_ttl= must not override that.
+                response.headers["Cache-Control"] = "private, no-store"
+                response.headers["Vary"] = "Cookie"
+            else:
                 ttl = int(ttl)
                 if ttl == 0:
                     ttl_header = "no-cache"
@@ -391,21 +407,27 @@ class RowView(BaseView):
         return response
 
     async def data(self, request, default_labels=False):
-        resolved = await self.ds.resolve_row(request)
-        db = resolved.db
+        db, table, resource = await _database_and_table_resource_from_request(
+            self.ds, request
+        )
         database = db.name
-        table = resolved.table
-        pk_values = resolved.pk_values
 
-        # Ensure user has permission to view this row
+        # Check the URL resource before resolving the row, so a denied request
+        # cannot distinguish an existing primary key from a missing one.
         visible, private = await self.ds.check_visibility(
             request.actor,
             action="view-table",
-            resource=TableResource(database=database, table=table),
+            resource=resource,
         )
         if not visible:
             raise Forbidden("You do not have permission to view this table")
+        # Record whether this response is private (visible to this actor
+        # only) so set_response_headers() can set appropriate Cache-Control
+        # headers, regardless of which output format ends up being rendered.
+        request._datasette_private_response = private
 
+        resolved = await self.ds.resolve_row(request)
+        pk_values = resolved.pk_values
         results = await resolved.db.execute(
             resolved.sql, resolved.params, truncate=True
         )
@@ -482,8 +504,8 @@ class RowView(BaseView):
             for row in display_rows:
                 for cell in row:
                     if cell["column"] in pk_set:
-                        cell["value"] = markupsafe.Markup(
-                            "<strong>{}</strong>".format(cell["value"])
+                        cell["value"] = markupsafe.Markup("<strong>{}</strong>").format(
+                            cell["value"]
                         )
 
             label_column = await db.label_column_for_table(table) if is_table else None
@@ -556,7 +578,7 @@ class RowView(BaseView):
                 "private": private,
                 "columns": reordered_columns,
                 "foreign_key_tables": await self.foreign_key_tables(
-                    database, table, pk_values
+                    database, table, pk_values, actor=request.actor
                 ),
                 "database_color": db.color,
                 "display_columns": display_columns,
@@ -633,12 +655,23 @@ class RowView(BaseView):
             ),
         )
 
-    async def foreign_key_tables(self, database, table, pk_values):
+    async def foreign_key_tables(self, database, table, pk_values, *, actor):
         if len(pk_values) != 1:
             return []
         db = self.ds.databases[database]
         all_foreign_keys = await db.get_all_foreign_keys()
-        foreign_keys = all_foreign_keys[table]["incoming"]
+        foreign_keys = []
+        table_permissions = {}
+        for fk in all_foreign_keys[table]["incoming"]:
+            other_table = fk["other_table"]
+            if other_table not in table_permissions:
+                table_permissions[other_table] = await self.ds.allowed(
+                    action="view-table",
+                    resource=TableResource(database=database, table=other_table),
+                    actor=actor,
+                )
+            if table_permissions[other_table]:
+                foreign_keys.append(fk)
         if len(foreign_keys) == 0:
             return []
 
@@ -695,9 +728,24 @@ def _truncated_row_flash_label(label):
     return label[: ROW_FLASH_LABEL_MAX_LENGTH - 1] + "\u2026"
 
 
-async def _row_flash_message(db, action, resolved, row=None):
+async def _row_flash_message(
+    datasette, request, action, resolved, row=None, *, refresh_row=False
+):
     pk_label = ", ".join(resolved.pk_values)
-    label_column = await db.label_column_for_table(resolved.table)
+    # Mutation permission does not grant access to stored row labels.
+    if not await datasette.allowed(
+        action="view-table",
+        resource=TableResource(database=resolved.db.name, table=resolved.table),
+        actor=request.actor,
+    ):
+        return f"{action} row {pk_label}"
+
+    if refresh_row and row is None:
+        results = await resolved.db.execute(
+            resolved.sql, resolved.params, truncate=True
+        )
+        row = results.first()
+    label_column = await resolved.db.label_column_for_table(resolved.table)
     label = row_label_from_label_column(row or resolved.row, label_column)
     if label:
         label = _truncated_row_flash_label(label)
@@ -710,22 +758,28 @@ async def _resolve_row_and_check_permission(datasette, request, permission):
     from datasette.app import DatabaseNotFound, RowNotFound, TableNotFound
 
     try:
-        resolved = await datasette.resolve_row(request)
+        _, _, resource = await _database_and_table_resource_from_request(
+            datasette, request
+        )
     except DatabaseNotFound as e:
         return False, Response.error([f"Database not found: {e.database_name}"], 404)
+
+    # Check the URL resource before resolving the row, so a denied request
+    # cannot distinguish an existing primary key from a missing one.
+    if not await datasette.allowed(
+        action=permission,
+        resource=resource,
+        actor=request.actor,
+    ):
+        return False, Response.error(["Permission denied"], 403)
+
+    try:
+        resolved = await datasette.resolve_row(request)
     except TableNotFound as e:
         return False, Response.error([f"Table not found: {e.table}"], 404)
     except RowNotFound as e:
         return False, Response.error([f"Record not found: {e.pk_values}"], 404)
 
-    # Ensure user has permission to delete this row
-    if not await datasette.allowed(
-        action=permission,
-        resource=TableResource(database=resolved.db.name, table=resolved.table),
-        actor=request.actor,
-    ):
-        return False, Response.error(["Permission denied"], 403)
-
     return True, resolved
 
 
@@ -744,6 +798,7 @@ class RowDeleteView(BaseView):
 
         # Delete table
         def delete_row(conn):
+            check_structured_write_table(conn, resolved.table)
             sqlite_utils.Database(conn)[resolved.table].delete(resolved.pk_values)
 
         try:
@@ -765,7 +820,7 @@ class RowDeleteView(BaseView):
             table_url = self.ds.urls.table(resolved.db.name, resolved.table)
             self.ds.add_message(
                 request,
-                await _row_flash_message(resolved.db, "Deleted", resolved),
+                await _row_flash_message(self.ds, request, "Deleted", resolved),
                 self.ds.INFO,
             )
             return Response.json({"ok": True, "redirect": str(table_url)}, status=200)
@@ -826,6 +881,7 @@ class RowUpdateView(BaseView):
             return Response.error(["Permission denied for alter-table"], 403)
 
         def update_row(conn):
+            check_structured_write_table(conn, resolved.table)
             sqlite_utils.Database(conn)[resolved.table].update(
                 resolved.pk_values, update, alter=alter
             )
@@ -838,7 +894,14 @@ class RowUpdateView(BaseView):
 
         result = {"ok": True}
         returned_row = None
-        if data.get("return"):
+        # Only read back and disclose the stored row if the actor is also
+        # allowed to view this table - update-row alone must not be usable
+        # to read data the actor cannot otherwise see.
+        if data.get("return") and await self.ds.allowed(
+            action="view-table",
+            resource=TableResource(database=resolved.db.name, table=resolved.table),
+            actor=request.actor,
+        ):
             results = await resolved.db.execute(
                 resolved.sql, resolved.params, truncate=True
             )
@@ -855,16 +918,15 @@ class RowUpdateView(BaseView):
         )
 
         if request.args.get("_message"):
-            message_row = returned_row
-            if message_row is None:
-                results = await resolved.db.execute(
-                    resolved.sql, resolved.params, truncate=True
-                )
-                message_row = results.first()
             self.ds.add_message(
                 request,
                 await _row_flash_message(
-                    resolved.db, "Updated", resolved, row=message_row
+                    self.ds,
+                    request,
+                    "Updated",
+                    resolved,
+                    row=returned_row,
+                    refresh_row=True,
                 ),
                 self.ds.INFO,
             )

datasette/views/special.py

@@ -311,6 +311,7 @@ class AllowedResourcesView(BaseView):
     has_json_alternate = False
 
     async def get(self, request):
+        await self.ds.ensure_permission(action="view-instance", actor=request.actor)
         await self.ds.refresh_schemas()
 
         # Check if user has permissions-debug (to show sensitive fields)
@@ -796,6 +797,8 @@ class CreateTokenView(BaseView):
             raise Forbidden(
                 "Token authentication cannot be used to create additional tokens"
             )
+        if "_r" in request.actor:
+            raise Forbidden("Restricted actors cannot create API tokens")
 
     async def shared(self, request):
         self.check_permission(request)
@@ -873,6 +876,11 @@ class CreateTokenView(BaseView):
                 else:
                     errors.append("Invalid expire duration unit")
 
+        if errors:
+            context = await self.shared(request)
+            context["errors"] = errors
+            return await self.render(["create_token.html"], request, context)
+
         # Are there any restrictions?
         from datasette.tokens import TokenRestrictions
 
@@ -1261,14 +1269,21 @@ class SchemaBaseView(BaseView):
 
     has_json_alternate = False
 
-    async def get_database_schema(self, database_name):
+    async def get_database_schema(self, database_name, actor):
         """Get schema SQL for a database."""
         db = self.ds.databases[database_name]
-        result = await db.execute(
-            "select group_concat(sql, ';' || CHAR(10)) as schema from sqlite_master where sql is not null"
+        allowed_tables_page = await self.ds.allowed_resources(
+            "view-table", actor, parent=database_name
+        )
+        allowed_table_names = {
+            resource.child async for resource in allowed_tables_page.all()
+        }
+        result = await db.execute(
+            "select tbl_name, sql from sqlite_master where sql is not null"
+        )
+        return ";\n".join(
+            row["sql"] for row in result.rows if row["tbl_name"] in allowed_table_names
         )
-        row = result.first()
-        return row["schema"] if row and row["schema"] else ""
 
     def format_json_response(self, data):
         """Format data as JSON response with CORS headers if needed."""
@@ -1330,7 +1345,7 @@ class InstanceSchemaView(SchemaBaseView):
         # Get schema for each database
         schemas = []
         for database_name in allowed_databases:
-            schema = await self.get_database_schema(database_name)
+            schema = await self.get_database_schema(database_name, request.actor)
             schemas.append({"database": database_name, "schema": schema})
 
         if format_ == "json":
@@ -1371,7 +1386,7 @@ class DatabaseSchemaView(SchemaBaseView):
         if database_name not in self.ds.databases:
             return self.format_error_response("Database not found", format_)
 
-        schema = await self.get_database_schema(database_name)
+        schema = await self.get_database_schema(database_name, request.actor)
 
         if format_ == "json":
             return self.format_json_response(
@@ -1410,7 +1425,8 @@ class TableSchemaView(SchemaBaseView):
         # Get schema for the table
         db = self.ds.databases[database_name]
         result = await db.execute(
-            "select sql from sqlite_master where name = ? and sql is not null",
+            "select sql from sqlite_master where name = ? "
+            "and type in ('table', 'view') and sql is not null",
             [table_name],
         )
         row = result.first()

datasette/views/stored_queries.py

@@ -279,7 +279,7 @@ class QueryCreateView(BaseView):
             ),
         )
         response.status = status
-        return response
+        return _block_framing(response)
 
     async def get(self, request):
         db = await self.ds.resolve_database(request)
@@ -527,7 +527,7 @@ class QueryEditView(BaseView):
             ),
         )
         response.status = status
-        return response
+        return _block_framing(response)
 
     async def get(self, request):
         db, query_name, existing = await self._load(request)
@@ -639,7 +639,8 @@ class QueryDeleteView(BaseView):
             return Response.error(
                 ["Trusted queries cannot be deleted using the API"], 403
             )
-        return await self.render(
+        return _block_framing(
+            await self.render(
                 ["query_delete.html"],
                 request,
                 {
@@ -649,6 +650,7 @@ class QueryDeleteView(BaseView):
                     "query_url": self.ds.urls.table(db.name, query_name),
                 },
             )
+        )
 
     async def post(self, request):
         db, query_name, existing = await self._load(request)

datasette/views/table.py

@@ -1,6 +1,7 @@
 import asyncio
 import itertools
 import json
+import time
 import urllib
 import urllib.parse
 from dataclasses import dataclass, field
@@ -57,6 +58,7 @@ from datasette.utils.asgi import (
     Request,
     Response,
 )
+from datasette.utils.sqlite import check_structured_write_table
 
 from . import Context, from_extra
 from .base import BaseView, DatasetteError, stream_csv
@@ -670,7 +672,7 @@ async def display_columns_and_rows(
     }
     pks = await db.primary_keys(table_name)
     pks_for_display = pks
-    if not pks_for_display:
+    if not pks_for_display and not await db.view_exists(table_name):
         pks_for_display = ["rowid"]
     label_column = None
     if link_column:
@@ -1126,6 +1128,7 @@ class TableInsertView(BaseView):
             row_pk_values_for_later = [tuple(row[pk] for pk in pks) for row in rows]
 
         def insert_or_upsert_rows(conn):
+            check_structured_write_table(conn, table_name)
             table = sqlite_utils.Database(conn)[table_name]
             kwargs = {}
             if upsert:
@@ -1157,17 +1160,32 @@ class TableInsertView(BaseView):
             # TODO: narrow to expected write errors so Datasette bugs surface as 500s
             return Response.error([str(e)])
         result = {"ok": True}
+        # Only read back and disclose stored rows if the actor is also
+        # allowed to view this table - insert-row/update-row alone must
+        # not be usable to read data the actor cannot otherwise see.
+        if should_return and not await self.ds.allowed(
+            action="view-table",
+            resource=TableResource(database=database_name, table=table_name),
+            actor=request.actor,
+        ):
+            should_return = False
         if should_return:
             if upsert:
                 # Fetch based on initial input IDs
                 where_clause = " OR ".join(
-                    ["({})".format(" AND ".join(f"{pk} = ?" for pk in pks))]
+                    [
+                        "({})".format(
+                            " AND ".join(f"{escape_sqlite(pk)} = ?" for pk in pks)
+                        )
+                    ]
                     * len(row_pk_values_for_later)
                 )
                 args = list(itertools.chain.from_iterable(row_pk_values_for_later))
                 fetched_rows = await db.execute(
-                    "select {}* from [{}] where {}".format(
-                        "rowid, " if pks == ["rowid"] else "", table_name, where_clause
+                    "select {}* from {} where {}".format(
+                        "rowid, " if pks == ["rowid"] else "",
+                        escape_sqlite(table_name),
+                        where_clause,
                     ),
                     args,
                 )
@@ -1382,7 +1400,9 @@ class TableDropView(BaseView):
                     "database": database_name,
                     "table": table_name,
                     "row_count": (
-                        await db.execute(f"select count(*) from [{table_name}]")
+                        await db.execute(
+                            f"select count(*) from {escape_sqlite(table_name)}"
+                        )
                     ).single_value(),
                     "message": 'Pass "confirm": true to confirm',
                 },
@@ -1409,6 +1429,42 @@ class TableDropView(BaseView):
         return Response.json({"ok": True}, status=200)
 
 
+class TableCountView(BaseView):
+    name = "table-count"
+
+    async def post(self, request):
+        try:
+            return await self.count(request)
+        except (NotFound, Forbidden, BadRequest, DatasetteError) as ex:
+            return Response.error(str(ex), status=ex.status)
+
+    async def count(self, request):
+        resolved = await self.ds.resolve_table(request)
+        visible, _private = await self.ds.check_visibility(
+            request.actor,
+            action="view-table",
+            resource=TableResource(database=resolved.db.name, table=resolved.table),
+        )
+        if not visible:
+            raise Forbidden("You do not have permission to view this table")
+        _, where_clauses, params, _, _ = await _table_filters(
+            self.ds, request, resolved.db.name, resolved.table
+        )
+        sql = f"select count(*) from {escape_sqlite(resolved.table)}"
+        if where_clauses:
+            sql += " where " + " and ".join(where_clauses)
+        try:
+            results = await resolved.db.execute(sql, params)
+        except QueryInterrupted:
+            return Response.error("Count query timed out", status=400)
+        except (sqlite3.OperationalError, InvalidSql) as ex:
+            return Response.error(str(ex), status=400)
+        return Response.json(
+            {"ok": True, "count": results.single_value()},
+            headers={"Cache-Control": "no-store"},
+        )
+
+
 class TableFragmentView(BaseView):
     name = "table-fragment"
 
@@ -1695,7 +1751,16 @@ async def table_view(datasette, request):
     if ttl is None or not ttl.isdigit():
         ttl = datasette.setting("default_cache_ttl")
 
+    private = getattr(request, "_datasette_private_response", False)
+
     if datasette.cache_headers and response.status == 200:
+        if private:
+            # This response is only visible to the current actor (denied to
+            # anonymous requests), so it must never be stored by a shared
+            # cache/CDN - and ?_ttl= must not be able to override that.
+            response.headers["Cache-Control"] = "private, no-store"
+            response.headers["Vary"] = "Cookie"
+        else:
             ttl = int(ttl)
             if ttl == 0:
                 ttl_header = "no-cache"
@@ -1737,6 +1802,7 @@ async def table_view_traced(datasette, request):
         context_for_html_hack = True
         default_labels = True
 
+    start = time.perf_counter()
     view_data = await table_view_data(
         datasette,
         request,
@@ -1747,6 +1813,7 @@ async def table_view_traced(datasette, request):
     )
     if isinstance(view_data, Response):
         return view_data
+    query_ms = (time.perf_counter() - start) * 1000
     data, rows, columns, _expanded_columns, sql, next_url = view_data
 
     # Handle formats from plugins
@@ -1893,7 +1960,7 @@ async def table_view_traced(datasette, request):
                 resource=DatabaseResource(database=resolved.db.name),
                 actor=request.actor,
             ),
-            query_ms=1.2,
+            query_ms=query_ms,
             select_templates=[
                 f"{'*' if template_name == template.name else ''}{template_name}"
                 for template_name in templates
@@ -1925,6 +1992,47 @@ async def table_view_traced(datasette, request):
     return r
 
 
+async def _table_filters(datasette, request, database_name, table_name):
+    # Arguments that start with _ and don't contain a __ are
+    # special - things like ?_search= - and should not be
+    # treated as filters.
+    filter_args = []
+    for key in request.args:
+        if not (key.startswith("_") and "__" not in key):
+            for v in request.args.getlist(key):
+                filter_args.append((key, v))
+
+    # Build where clauses from query string arguments
+    filters = Filters(sorted(filter_args))
+    where_clauses, params = filters.build_where_clauses(table_name)
+
+    # Execute filters_from_request plugin hooks - including the default
+    # ones that live in datasette/filters.py
+    extra_context_from_filters = {}
+    extra_human_descriptions = []
+
+    for hook in pm.hook.filters_from_request(
+        request=request,
+        table=table_name,
+        database=database_name,
+        datasette=datasette,
+    ):
+        filter_arguments = await await_me_maybe(hook)
+        if filter_arguments:
+            where_clauses.extend(filter_arguments.where_clauses)
+            params.update(filter_arguments.params)
+            extra_human_descriptions.extend(filter_arguments.human_descriptions)
+            extra_context_from_filters.update(filter_arguments.extra_context)
+
+    return (
+        filters,
+        where_clauses,
+        params,
+        extra_human_descriptions,
+        extra_context_from_filters,
+    )
+
+
 async def table_view_data(
     datasette,
     request,
@@ -1949,6 +2057,10 @@ async def table_view_data(
     )
     if not visible:
         raise Forbidden("You do not have permission to view this table")
+    # Record whether this response is private (visible to this actor only)
+    # so the outer table_view() can set appropriate Cache-Control headers,
+    # regardless of which output format ends up being rendered.
+    request._datasette_private_response = private
 
     # Redirect based on request.args, if necessary
     redirect_response = await _redirect_if_needed(datasette, request, resolved)
@@ -1999,36 +2111,13 @@ async def table_view_data(
 
     table_metadata = await datasette.table_config(database_name, table_name)
 
-    # Arguments that start with _ and don't contain a __ are
-    # special - things like ?_search= - and should not be
-    # treated as filters.
-    filter_args = []
-    for key in request.args:
-        if not (key.startswith("_") and "__" not in key):
-            for v in request.args.getlist(key):
-                filter_args.append((key, v))
-
-    # Build where clauses from query string arguments
-    filters = Filters(sorted(filter_args))
-    where_clauses, params = filters.build_where_clauses(table_name)
-
-    # Execute filters_from_request plugin hooks - including the default
-    # ones that live in datasette/filters.py
-    extra_context_from_filters = {}
-    extra_human_descriptions = []
-
-    for hook in pm.hook.filters_from_request(
-        request=request,
-        table=table_name,
-        database=database_name,
-        datasette=datasette,
-    ):
-        filter_arguments = await await_me_maybe(hook)
-        if filter_arguments:
-            where_clauses.extend(filter_arguments.where_clauses)
-            params.update(filter_arguments.params)
-            extra_human_descriptions.extend(filter_arguments.human_descriptions)
-            extra_context_from_filters.update(filter_arguments.extra_context)
+    (
+        filters,
+        where_clauses,
+        params,
+        extra_human_descriptions,
+        extra_context_from_filters,
+    ) = await _table_filters(datasette, request, database_name, table_name)
 
     # Deal with custom sort orders
     sortable_columns = await _sortable_columns_for_table(
@@ -2226,8 +2315,6 @@ async def table_view_data(
                 new_rows.append(new_row)
             rows = new_rows
 
-    _next = request.args.get("_next")
-
     # Pagination next link
     next_value, next_url = await _next_value_and_url(
         datasette,
@@ -2432,9 +2519,12 @@ async def _next_value_and_url(
             except IndexError:
                 # sort/sort_desc column missing from SELECT - look up value by PK instead
                 prefix_where_clause = " and ".join(
-                    f"[{pk}] = :pk{i}" for i, pk in enumerate(pks)
+                    f"{escape_sqlite(pk)} = :pk{i}" for i, pk in enumerate(pks)
+                )
+                prefix_lookup_sql = (
+                    f"select {escape_sqlite(sort or sort_desc)} "
+                    f"from {escape_sqlite(table_name)} where {prefix_where_clause}"
                 )
-                prefix_lookup_sql = f"select [{sort or sort_desc}] from [{table_name}] where {prefix_where_clause}"
                 prefix = (
                     await db.execute(
                         prefix_lookup_sql,

datasette/views/table_create_alter.py

@@ -27,7 +27,15 @@ from datasette.utils import (
     table_column_details,
 )
 from datasette.utils.asgi import NotFound, PayloadTooLarge, Response
-from datasette.utils.sqlite import sqlite_hidden_table_names
+from datasette.utils.permissions import (
+    SKIP_PERMISSION_CHECKS,
+    gather_permission_sql_from_hooks,
+    resolve_permissions_with_candidates,
+)
+from datasette.utils.sqlite import (
+    check_structured_write_table,
+    sqlite_hidden_table_names,
+)
 
 from .base import BaseView
 
@@ -122,6 +130,30 @@ def _public_foreign_key_target(target):
     }
 
 
+async def _filter_visible_foreign_key_targets(datasette, actor, database_name, targets):
+    if not targets:
+        return []
+
+    permission_sqls = await gather_permission_sql_from_hooks(
+        datasette=datasette,
+        actor=actor,
+        action="view-table",
+    )
+    if permission_sqls is SKIP_PERMISSION_CHECKS:
+        return targets
+
+    candidate_tables = list(dict.fromkeys(target["fk_table"] for target in targets))
+    permission_rows = await resolve_permissions_with_candidates(
+        datasette.get_internal_database(),
+        actor,
+        permission_sqls,
+        [(database_name, table_name) for table_name in candidate_tables],
+        "view-table",
+    )
+    visible_tables = {row["child"] for row in permission_rows if bool(row["allow"])}
+    return [target for target in targets if target["fk_table"] in visible_tables]
+
+
 def _singular(name):
     if name.endswith("ies") and len(name) > 3:
         return name[:-3] + "y"
@@ -821,16 +853,18 @@ class TableCreateView(BaseView):
         ignore = create_request.ignore
         replace = create_request.replace
 
+        table_name = create_request.table
+        table_exists = await db.table_exists(table_name)
+        table_resource = TableResource(database=database_name, table=table_name)
+
         # Replacing rows requires update-row permission
         if replace and not await self.ds.allowed(
             action="update-row",
-            resource=DatabaseResource(database=database_name),
+            resource=table_resource,
             actor=request.actor,
         ):
             return Response.error(["Permission denied: need update-row"], 403)
 
-        table_name = create_request.table
-        table_exists = await db.table_exists(table_name)
         columns = create_request.columns
         rows = create_request.rows_list
 
@@ -838,7 +872,7 @@ class TableCreateView(BaseView):
             # Must have insert-row permission
             if not await self.ds.allowed(
                 action="insert-row",
-                resource=DatabaseResource(database=database_name),
+                resource=table_resource,
                 actor=request.actor,
             ):
                 return Response.error(["Permission denied: need insert-row"], 403)
@@ -857,7 +891,7 @@ class TableCreateView(BaseView):
                 if create_request.alter:
                     if not await self.ds.allowed(
                         action="alter-table",
-                        resource=DatabaseResource(database=database_name),
+                        resource=table_resource,
                         actor=request.actor,
                     ):
                         return Response.error(
@@ -893,6 +927,7 @@ class TableCreateView(BaseView):
             )
 
         def create_table(conn):
+            check_structured_write_table(conn, table_name, allow_missing=True)
             db_for_write = sqlite_utils.Database(conn)
             table = db_for_write[table_name]
             if rows:
@@ -1012,6 +1047,9 @@ class DatabaseForeignKeyTargetsView(BaseView):
             for target in (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts()
             if target["fk_table"] not in hidden_tables
         ]
+        targets = await _filter_visible_foreign_key_targets(
+            self.ds, request.actor, database_name, targets
+        )
         return Response.json(
             {
                 "ok": True,
@@ -1050,6 +1088,15 @@ class TableForeignKeySuggestionsView(BaseView):
         source_columns, targets, current_by_column = await db.execute_fn(
             lambda conn: _foreign_key_suggestion_metadata(conn, table_name)
         )
+        targets = await _filter_visible_foreign_key_targets(
+            self.ds, request.actor, database_name, targets
+        )
+        visible_target_tables = {target["fk_table"] for target in targets}
+        current_by_column = {
+            column: current
+            for column, current in current_by_column.items()
+            if current["fk_table"] in visible_target_tables
+        }
 
         columns = []
         options_by_column = {}
@@ -1262,7 +1309,9 @@ class TableAlterView(BaseView):
                     elif operation.op == "set_foreign_keys":
                         foreign_keys = [fk.tuple for fk in args.foreign_keys]
 
-                with operation_conn:
+                # Use a savepoint inside execute_write_fn's transaction so
+                # write_wrapper hooks can still reject and roll back the write.
+                with db_for_write.atomic():
                     for column in add_columns:
                         not_null_default = None
                         if column.not_null:

datasette/views/table_extras.py

@@ -1206,7 +1206,10 @@ class ForeignKeyTablesExtra(Extra):
 
     async def resolve(self, context):
         return await context.foreign_key_tables(
-            context.database_name, context.table_name, context.pk_values
+            context.database_name,
+            context.table_name,
+            context.pk_values,
+            actor=context.request.actor,
         )
 
 

datasette/write_sql.py

@@ -83,6 +83,22 @@ def decision_for_write_sql_operation(
             )
     if operation.operation == "function":
         return IgnoreWriteSqlOperation("SQL function")
+    if (
+        operation.operation == "read"
+        and operation.target_type == "table"
+        and operation.table is not None
+        and operation.table_kind is None
+        and operation.table.lower().startswith("pragma_")
+    ):
+        # Eponymous table-valued PRAGMA functions (e.g. pragma_table_info("secret"))
+        # report a read of the synthetic "pragma_table_info" table, not of the
+        # table passed as an argument. That means a view-table denial on the real
+        # table is never consulted, so these could otherwise be used to read
+        # schema metadata (column names, table lists, ...) for tables the actor
+        # is not allowed to view. Reject them outright in untrusted write SQL,
+        # including inside CREATE VIEW bodies (whose reads are discovered here
+        # via the rolled-back dependency-read analysis above).
+        return UnsupportedWriteSqlOperation(unsupported_message)
     if (
         operation.operation == "read"
         and operation.target_type == "table"

docs/authentication.rst

@@ -158,6 +158,15 @@ Datasette resolves matching rules from most specific to least specific:
 
 This means a resource-level allow can provide an exception to a parent-level deny. It also means that two plugins which disagree at the same level resolve to deny.
 
+For table and view permissions, resource names use SQLite's case-insensitive
+identifier matching: ``Secret``, ``secret`` and ``SECRET`` identify the same
+table. This applies to configuration rules, plugin rules and token restrictions.
+Only ASCII letters are case-insensitive; non-ASCII characters remain distinct.
+Conflicting rules for different spellings of the same name follow the usual
+deny-wins rule at the same scope. Names retain their original spelling in
+resource listings and permission explanations. Database names, stored query
+names and other resource types remain case-sensitive.
+
 .. list-table:: Permission rule examples
    :header-rows: 1
 
@@ -182,6 +191,18 @@ This means a resource-level allow can provide an exception to a parent-level den
 
 The built-in public defaults are global allow rules for actions such as ``view-instance``, ``view-database`` and ``view-table``. They follow the same precedence rules as configuration and plugin rules. The ``--default-deny`` option prevents Datasette from contributing those default allow rules.
 
+The built-in ``datasette.default_permissions.sqlite_statistics`` plugin denies
+``view-table`` for ``sqlite_stat1``, ``sqlite_stat2``, ``sqlite_stat3`` and
+``sqlite_stat4``. These table-level denials also apply to root users and take
+precedence over configuration or plugin allow rules at the same scope.
+This controls table access and listings, without changing ``execute-sql`` or
+SQLite's internal use of statistics.
+
+A plugin can replace this policy by unregistering
+``datasette.default_permissions.sqlite_statistics`` through ``datasette.pm``
+and registering its own permission hook. Plugin registration is process-wide:
+replacing this policy affects every Datasette instance in that process.
+
 Datasette performs checks using :ref:`datasette_allowed`, which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``.
 
 ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified.
@@ -771,6 +792,8 @@ Datasette defaults to allowing any site visitor to execute their own custom SQL
 
 Access to this ability is controlled by the :ref:`actions_execute_sql` permission.
 
+This permission does not apply to structured table-browsing operations where Datasette constructs the SQL, such as sorting, column filters and :ref:`facets`. Faceting is controlled separately by the :ref:`setting_allow_facet` setting.
+
 The easiest way to disable arbitrary SQL queries is using the :ref:`default_allow_sql setting <setting_default_allow_sql>` when you first start Datasette running.
 
 You can alternatively use an ``"allow_sql"`` block to control who is allowed to execute arbitrary SQL queries.
@@ -1359,6 +1382,12 @@ view-table
 
 Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.io/fixtures/complex_foreign_keys
 
+Derived implementation tables require access to their immediate source: FTS and RTree shadow tables require access to their virtual table, external-content FTS tables require access to their content table, and FTS vocabulary tables (``fts5vocab`` and ``fts4aux``) require access to their FTS table. The derived table's own permission rules also apply.
+
+Access is always denied if the source table is itself derived, or if a vocabulary table's source cannot be identified.
+
+The same rules apply to individual permission checks and table listings, including whether they are private. If a database error prevents dependency discovery, the check or listing fails with an error instead of ignoring the dependencies. Failed discovery results are not cached, so later checks can retry.
+
 ``resource`` - ``datasette.resources.TableResource(database, table)``
     ``database`` is the name of the database (string)
 
@@ -1521,6 +1550,8 @@ execute-sql
 
 Actor is allowed to run arbitrary read-only SQL queries against a specific database using the :ref:`custom SQL query page <pages_custom_sql_queries>`, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100
 
+This action also controls raw SQL supplied using ``?_where=``. It does not control structured table-browsing features such as :ref:`facets`, which use SQL generated by Datasette and are controlled by :ref:`setting_allow_facet`.
+
 ``resource`` - ``datasette.resources.DatabaseResource(database)``
     ``database`` is the name of the database (string)
 

docs/changelog.rst

@@ -4,6 +4,89 @@
 Changelog
 =========
 
+.. _v1_0_a40:
+
+1.0a40 (2026-09-16)
+-------------------
+
+A security fix, a new set of APIs providing background tasks for plugins, an endpoint for counting matching rows, and a collection of bug fixes.
+
+Security fix
+~~~~~~~~~~~~
+
+- Fixed a security issue where a trailing newline in a requested table name could bypass table permissions and expose private rows. Thanks for the report, `dpfkdlemtp <https://github.com/dpfkdlemtp>`__. `GHSA-h547-rmjf-5m2m <https://github.com/simonw/datasette/security/advisories/GHSA-h547-rmjf-5m2m>`__
+
+Background tasks
+~~~~~~~~~~~~~~~~
+
+Datasette plugins can now use **background tasks** to run code independent of the Datasette request/response cycle.
+
+- New :ref:`datasette_add_background_task` API: plugins register supervised, long-lived background work - typically from a ``startup`` hook - and these will be launched after every ``startup`` hook has run. Tasks are cancelled (with a five-second grace period) on shutdown.
+- New ``/-/tasks`` JSON debug endpoint lists every supervised background task and its state, in the style of ``/-/threads``. See :ref:`JsonDataView_tasks`. It requires the ``permissions-debug`` permission.
+- New :ref:`plugin_hook_shutdown` plugin hook, called during graceful shutdown (Ctrl-C, ``SIGTERM``) before background tasks are cancelled and before database connections are closed. It is not called on a hard kill (``SIGKILL``).
+- Plugin ``asgi_wrapper`` middleware now always runs *after* startup has completed.
+- If your plugin uses ``asgi_wrapper`` to start background tasks on the first incoming request, you should migrate to ``datasette.add_background_task()`` instead. `datasette-cron <https://datasette.io/plugins/datasette-cron>`__ and `datasette-enrichments <https://datasette.io/plugins/datasette-enrichments>`__ are being migrated to this pattern.
+
+Other features
+~~~~~~~~~~~~~~
+
+- New :ref:`POST count endpoint <TableCountView>` for counting filtered table rows, now used by the **count all** button. (:issue:`2914`)
+- Datasette now uses `httpx2 <https://httpx2.pydantic.dev/>`__, the Pydantic-maintained continuation of `httpx <https://www.python-httpx.org/>`__, in place of ``httpx``. The public API is the same, but responses returned by :ref:`internals_datasette_client` are now ``httpx2.Response`` objects rather than ``httpx.Response``. Plugins that use ``isinstance()`` checks against ``httpx.Response`` should be updated to use ``httpx2``. **Plugins that use httpx without explicitly depending on it** will need to add an explicit dependency or switch to `httpx2`.
+
+Bug fixes
+~~~~~~~~~
+
+- Column facets now show the remove-filter link for filters using ``column__exact=value``, as well as ``column=value``. (:issue:`1695`)
+- The :ref:`alter-table API <TableAlterView>` now rolls back schema changes when a :ref:`write_wrapper <plugin_hook_write_wrapper>` raises after the write. (:issue:`2924`, :pr:`2925`)
+- The :ref:`extra_template_vars() <plugin_hook_extra_template_vars>` plugin hook can now return a function or awaitable that resolves to ``None`` when no extra variables are needed. (:issue:`2005`)
+- :ref:`request.headers <internals_request>` now supports case-insensitive header lookups, so ``request.headers.get("Content-Type")`` works as well as ``request.headers.get("content-type")``. (:issue:`1861`)
+- CSV endpoints now return plain-text error messages for SQL errors. (:issue:`2129`)
+- The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook now receives an empty ``pks`` list when rendering SQL views in HTML, matching the JSON ``?_extra=render_cell`` behavior. (:issue:`2639`)
+- Numeric comparison filters now correctly handle decimal values, negative numbers and scientific notation when filtering computed columns and SQL views. Thanks, `Rami Abdelrazzaq <https://github.com/RamiNoodle733>`__. (:issue:`1681`, :pr:`2876`)
+- Fixed CSV streaming with ``?_stream=on`` on SQL views repeating the second page of results until the CSV size limit was reached. Thanks, `Ankita Advitot <https://github.com/AnkitaAdvitot>`__. (:issue:`2902`, :pr:`2903`)
+
+.. _v1_0_a39:
+
+1.0a39 (2026-09-10)
+-------------------
+
+This alpha release includes security fixes for permissions, SQL construction, HTML rendering, authentication and caching, plus improvements to application startup and write execution.
+
+See `0.65.4 <https://docs.datasette.io/en/stable/changelog.html#v0-65-4>`__ for fixes that have been backported to the stable 0.65.x branch.
+
+The Datasette blog `has more details on these releases <https://datasette.io/blog/2026/september-security-releases/>`__.
+
+Some of the security fixes include:
+
+- Table and view permission checks now take SQLite's case-insensitive names into account. See :ref:`authentication_permissions_explained`.
+- Viewing a full-text search index table now checks you have permission to view the table from which it draws its content.
+- Viewing SQLite statistics tables (``sqlite_stat1`` through ``sqlite_stat4``) is now denied by a default.
+- Table schema display now obeys the ``view-table`` permission.
+- Table filters using ``?_through=`` require permission to view the intermediate table.
+- Foreign-key target and suggestion APIs, incoming foreign-key relationships and their row counts now respect ``view-table`` permission.
+- Row endpoints check permissions before resolving primary keys, to avoid revealing the existence of an otherwise invisible primary key.
+- Improved permission checks for the create-table API. See :ref:`json_api_write`.
+- The write SQL interface now checks ``view-table`` permission for tables referenced by ``CREATE VIEW`` statements.
+- Fixed SQL identifier escaping for column names from untrusted database schemas.
+- Fixed HTML escaping for column names from untrusted database schemas.
+- URL columns now render links only for validated HTTP or HTTPS URLs.
+- Private and personalized dynamic responses now use ``Cache-Control: private, no-store``. Anonymous dynamic responses vary by ``Cookie`` and ``Authorization``.
+- Actor cookies now respect ``expire_after``.
+- Restricted actors can no longer create API tokens.
+- Stored-query create, edit and delete forms now block framing to prevent clickjacking.
+- Configuration secret redaction now matches key names case-insensitively.
+- SQLite extension loading is disabled after extensions supplied using ``--load-extension`` have been loaded.
+
+Other improvements and fixes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- :ref:`db.execute_write() <database_execute_write>` now has a default execution time limit of 2,000ms. Plugins can override this using ``time_limit_ms=`` or disable it using ``time_limit_ms=None``. This limit is independent of the ``sql_time_limit_ms`` setting for read queries.
+- Application startup now runs through ASGI lifespan events before requests are accepted, with a first-request fallback for hosts without lifespan support. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:pr:`2887`)
+- ``datasette serve`` now runs startup hooks and Uvicorn on the same event loop, preserving background tasks started by plugins. The minimum Uvicorn version is now 0.29. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:pr:`2886`)
+- Non-blocking writes using ``execute_write_fn(..., block=False)`` now return a distinct task UUID for every call and work correctly with ``num_sql_threads=0``. Thanks, `Zain Dana Harper <https://github.com/HarperZ9>`__. (:issue:`2860`, :issue:`2859`)
+- Dropping a table now disables its full-text search index first. (:issue:`2874`)
+- Fixed ``CREATE VIEW`` SQL analysis on Python 3.10.
+
 .. _v1_0_a38:
 
 1.0a38 (2026-08-06)
@@ -1092,7 +1175,7 @@ Features
 - New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`)
 - Spaces in the database names in URLs are now encoded as ``+`` rather than ``~20``. (:issue:`1701`)
 - ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`)
-- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image <https://hub.docker.com/datasetteproject/datasette>`__ has been upgraded to ``3.10.6-slim-bullseye``.  (:issue:`1768`)
+- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image <https://hub.docker.com/r/datasetteproject/datasette>`__ has been upgraded to ``3.10.6-slim-bullseye``.  (:issue:`1768`)
 - Canned writable queries against immutable databases now show a warning message. (:issue:`1728`)
 - ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (:pr:`1717`)
 - ``datasette publish cloudrun`` has new ``--min-instances`` and ``--max-instances`` options. (:issue:`1779`)
@@ -2121,7 +2204,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you
 - Removed obsolete ``?_group_count=col`` feature (:issue:`504`)
 - Improved user interface and documentation for ``datasette publish cloudrun`` (:issue:`608`)
 - Tables with indexes now show the ``CREATE INDEX`` statements on the table page (:issue:`618`)
-- Current version of `uvicorn <https://www.uvicorn.org/>`__ is now shown on ``/-/versions``
+- Current version of `uvicorn <https://uvicorn.dev/>`__ is now shown on ``/-/versions``
 - Python 3.8 is now supported! (:issue:`622`)
 - Python 3.5 is no longer supported.
 
@@ -2172,7 +2255,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you
 0.29.2 (2019-07-13)
 -------------------
 
-- Bumped `Uvicorn <https://www.uvicorn.org/>`__ to 0.8.4, fixing a bug where the query string was not included in the server logs. (:issue:`559`)
+- Bumped `Uvicorn <https://uvicorn.dev/>`__ to 0.8.4, fixing a bug where the query string was not included in the server logs. (:issue:`559`)
 - Fixed bug where the navigation breadcrumbs were not displayed correctly on the page for a custom query. (:issue:`558`)
 - Fixed bug where custom query names containing unicode characters caused errors.
 
@@ -2194,7 +2277,7 @@ ASGI, new plugin hooks, facet by date and much, much more...
 ASGI
 ~~~~
 
-`ASGI <https://asgi.readthedocs.io/>`__ is the Asynchronous Server Gateway Interface standard. I've been wanting to convert Datasette into an ASGI application for over a year - `Port Datasette to ASGI #272 <https://github.com/simonw/datasette/issues/272>`__ tracks thirteen months of intermittent development - but with Datasette 0.29 the change is finally released. This also means Datasette now runs on top of `Uvicorn <https://www.uvicorn.org/>`__ and no longer depends on `Sanic <https://github.com/huge-success/sanic>`__.
+`ASGI <https://asgi.readthedocs.io/>`__ is the Asynchronous Server Gateway Interface standard. I've been wanting to convert Datasette into an ASGI application for over a year - `Port Datasette to ASGI #272 <https://github.com/simonw/datasette/issues/272>`__ tracks thirteen months of intermittent development - but with Datasette 0.29 the change is finally released. This also means Datasette now runs on top of `Uvicorn <https://uvicorn.dev/>`__ and no longer depends on `Sanic <https://github.com/huge-success/sanic>`__.
 
 I wrote about the significance of this change in `Porting Datasette to ASGI, and Turtles all the way down <https://simonwillison.net/2019/Jun/23/datasette-asgi/>`__.
 
@@ -2608,7 +2691,7 @@ Miscellaneous
   as a string.
 * If you just want an array of the first value of each row, use the new
   ``?_shape=arrayfirst`` option - `example
-  <https://latest.datasette.io/fixtures.json?sql=select+neighborhood+from+facetable+order+by+pk+limit+101&_shape=arrayfirst>`_.
+  <https://latest.datasette.io/fixtures.json?sql=select+_neighborhood+from+facetable+order+by+pk+limit+101&_shape=arrayfirst>`_.
 
 0.22.1 (2018-05-23)
 -------------------

docs/cli-reference.rst

@@ -76,6 +76,8 @@ Or since this is the default command you can run this instead::
 
 Once started you can access it at ``http://localhost:8001``
 
+Use ``--internal PATH`` or the ``DATASETTE_INTERNAL`` environment variable to persist :ref:`Datasette's internal database <internals_internal>` to a SQLite file.
+
 .. [[[cog
     help(["serve", "--help"])
 .. ]]]

docs/custom_templates.rst

@@ -302,6 +302,14 @@ content you can do so by creating a ``row.html`` template like this:
 Note the ``default:row.html`` template name, which ensures Jinja will inherit
 from the default template.
 
+The default ``base.html`` template provides a ``crumbs`` block inside its navigation block. Override ``crumbs`` to customize the breadcrumbs without replacing the rest of the navigation. The imported ``crumbs.nav()`` macro renders Datasette's permission-aware breadcrumbs:
+
+.. code-block:: jinja
+
+    {% block crumbs %}
+    {{ crumbs.nav(request=request, database=database, table=table) }}
+    {% endblock %}
+
 The ``_table.html`` template is included by both the row and the table pages,
 and a list of rows. The default ``_table.html`` template renders them as an
 HTML template and `can be seen here <https://github.com/simonw/datasette/blob/main/datasette/templates/_table.html>`_.

docs/facets.rst

@@ -7,13 +7,15 @@ Datasette facets can be used to add a faceted browse interface to any database t
 With facets, tables are displayed along with a summary showing the most common values in specified columns.
 These values can be selected to further filter the table.
 
-Here's `an example <https://congress-legislators.datasettes.com/legislators/legislator_terms?_facet=type&_facet=party&_facet=state&_facet_size=10>`__:
+Here's `an example <https://datasette.io/legislators/legislator_terms?_facet=type&_facet=party&_facet=state&_facet_size=10>`__:
 
 .. image:: https://raw.githubusercontent.com/simonw/datasette-screenshots/0.62/non-retina/faceting-details.png
    :alt: Screenshot showing facets against a table of congressional legislators. Suggested facets include state_rank and start and end dates, and the displayed facets are state, party and type. Each facet lists values along with a count of rows for each value.
 
 Facets can be specified in two ways: using query string parameters, or in ``metadata.json`` configuration for the table.
 
+Facet queries are generated by Datasette and summarize rows the actor already has permission to view. They do not require the :ref:`actions_execute_sql` permission. Use the :ref:`setting_allow_facet` setting to control whether users can request facets using query string parameters.
+
 Facets in query strings
 -----------------------
 

docs/full_text_search.rst

@@ -19,7 +19,7 @@ The table page and table view API
 
 Table views that support full-text search can be queried using the ``?_search=TERMS`` query string parameter. This will run the search against content from all of the columns that have been included in the index.
 
-Try this example: `fara.datasettes.com/fara/FARA_All_ShortForms?_search=manafort <https://fara.datasettes.com/fara/FARA_All_ShortForms?_search=manafort>`__
+Try `searching Datasette ecosystem repositories for "csv" <https://datasette.io/content/repos?_search=csv>`__ to find tools for working with CSV files.
 
 SQLite full-text search supports wildcards. This means you can easily implement prefix auto-complete by including an asterisk at the end of the search term - for example::
 
@@ -52,11 +52,11 @@ Configuring full-text search for a table or view
 
 If a table has a corresponding FTS table set up using the ``content=`` argument to ``CREATE VIRTUAL TABLE`` shown below, Datasette will detect it automatically and add a search interface to the table page for that table.
 
-You can also manually configure which table should be used for full-text search using query string parameters or table configuration in ``datasette.yaml`` (see :ref:`table_configuration_fts`). You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option.
+You can also manually configure which table should be used for full-text search using table configuration in ``datasette.yaml`` (see :ref:`table_configuration_fts`). You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option.
 
-Use ``?_fts_table=x`` to over-ride the FTS table for a specific page. If the primary key was something other than ``rowid`` you can use ``?_fts_pk=col`` to set that as well. This is particularly useful for views, for example:
+The legacy ``?_fts_table=x`` and ``?_fts_pk=col`` query string parameters are accepted only if they exactly match the configured or automatically detected FTS mapping. They cannot be used to select a different FTS table or primary key. This prevents a public table from being used to probe the contents of a private FTS table.
 
-https://latest.datasette.io/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk
+Searching also requires the current actor to have ``view-table`` permission for the FTS table itself, in addition to permission to view the table or view being searched.
 
 The ``fts_table`` metadata property can be used to specify an associated FTS table. If the primary key column in your table which was used to populate the FTS table is something other than ``rowid``, you can specify the column to use with the ``fts_pk`` property.
 
@@ -120,41 +120,31 @@ Searches using custom SQL
 
 You can include full-text search results in custom SQL queries. The general pattern with SQLite search is to run the search as a sub-select that returns rowid values, then include those rowids in another part of the query.
 
-You can see the syntax for a basic search by running that search on a table page and then clicking "View and edit SQL" to see the underlying SQL. For example, consider this search for `manafort is the US FARA database <https://fara.datasettes.com/fara/FARA_All_ShortForms?_search=manafort>`_::
+You can see the syntax for a basic search by running that search on a table page and then clicking "View and edit SQL" to see the underlying SQL. For example, consider this search for `repositories mentioning "csv" <https://datasette.io/content/repos?_search=csv>`_::
 
-    /fara/FARA_All_ShortForms?_search=manafort
+    /content/repos?_search=csv
 
-If you click `View and edit SQL <https://fara.datasettes.com/fara?sql=select%0D%0A++rowid%2C%0D%0A++Short_Form_Termination_Date%2C%0D%0A++Short_Form_Date%2C%0D%0A++Short_Form_Last_Name%2C%0D%0A++Short_Form_First_Name%2C%0D%0A++Registration_Number%2C%0D%0A++Registration_Date%2C%0D%0A++Registrant_Name%2C%0D%0A++Address_1%2C%0D%0A++Address_2%2C%0D%0A++City%2C%0D%0A++State%2C%0D%0A++Zip%0D%0Afrom%0D%0A++FARA_All_ShortForms%0D%0Awhere%0D%0A++rowid+in+%28%0D%0A++++select%0D%0A++++++rowid%0D%0A++++from%0D%0A++++++FARA_All_ShortForms_fts%0D%0A++++where%0D%0A++++++FARA_All_ShortForms_fts+match+escape_fts%28%3Asearch%29%0D%0A++%29%0D%0Aorder+by%0D%0A++rowid%0D%0Alimit%0D%0A++101&search=manafort>`_ you'll see that the underlying SQL looks like this:
+The generated SQL selects all columns in the table. This simplified version selects just the repository ID, full name and description, using the same full-text search condition. `Run this query <https://datasette.io/content?sql=select%0A++id%2C%0A++full_name%2C%0A++description%0Afrom%0A++repos%0Awhere%0A++rowid+in+%28%0A++++select%0A++++++rowid%0A++++from%0A++++++repos_fts%0A++++where%0A++++++repos_fts+match+escape_fts%28%3Asearch%29%0A++%29%0Aorder+by%0A++id%0Alimit%0A++101&search=csv>`_ with ``search`` set to ``csv``:
 
 .. code-block:: sql
 
     select
-      rowid,
-      Short_Form_Termination_Date,
-      Short_Form_Date,
-      Short_Form_Last_Name,
-      Short_Form_First_Name,
-      Registration_Number,
-      Registration_Date,
-      Registrant_Name,
-      Address_1,
-      Address_2,
-      City,
-      State,
-      Zip
+      id,
+      full_name,
+      description
     from
-      FARA_All_ShortForms
+      repos
     where
       rowid in (
         select
           rowid
         from
-          FARA_All_ShortForms_fts
+          repos_fts
         where
-          FARA_All_ShortForms_fts match escape_fts(:search)
+          repos_fts match escape_fts(:search)
       )
     order by
-      rowid
+      id
     limit
       101
 

docs/internals.rst

@@ -26,7 +26,7 @@ The request object is passed to various plugin hooks. It represents an incoming
     The request scheme - usually ``https`` or ``http``.
 
 ``.headers`` - dictionary (str -> str)
-    A dictionary of incoming HTTP request headers. Header names have been converted to lowercase.
+    A dictionary of incoming HTTP request headers. Header lookups using ``request.headers["Content-Type"]``, ``request.headers.get("Content-Type")`` and ``"Content-Type" in request.headers`` are case-insensitive. Header names are lowercase when iterating over the dictionary.
 
 ``.cookies`` - dictionary (str -> str)
     A dictionary of incoming cookies
@@ -147,7 +147,7 @@ And a class method that can be used to create fake request objects for use in te
 .. _internals_multiparams:
 
 The MultiParams class
-=====================
+---------------------
 
 ``request.args`` is a ``MultiParams`` object - a dictionary-like object which provides access to query string parameters that may have multiple values.
 
@@ -177,7 +177,7 @@ Consider the query string ``?foo=1&foo=2&bar=3`` - with two values for ``foo`` a
 .. _internals_formdata:
 
 The FormData class
-==================
+------------------
 
 ``await request.form()`` returns a ``FormData`` object - a dictionary-like object which provides access to form fields and uploaded files. It has a similar interface to ``MultiParams``.
 
@@ -205,7 +205,7 @@ The FormData class
 .. _internals_uploadedfile:
 
 The UploadedFile class
-======================
+----------------------
 
 When parsing multipart form data with ``files=True``, file uploads are returned as ``UploadedFile`` objects with the following properties and methods:
 
@@ -1403,7 +1403,31 @@ Release all resources held by this ``Datasette`` instance. This calls :ref:`data
 
 If a call to ``Database.close()`` on one of the attached databases raises an exception, ``Datasette.close()`` will continue trying to close the remaining databases and will re-raise the first exception after every database has been processed.
 
-When Datasette is being served over ASGI the ``close()`` method is wired up to the lifespan shutdown event, so resources are released cleanly on ``SIGTERM`` / ``SIGINT``.
+When Datasette is being served over ASGI the ``close()`` method is wired up to the lifespan shutdown event, so resources are released cleanly on ``SIGTERM`` / ``SIGINT``. See :ref:`datasette_lifecycle` for where ``close()`` fits into the full startup-to-shutdown sequence.
+
+.. _datasette_add_background_task:
+
+.add_background_task(func, name=None)
+-------------------------------------
+
+``func`` - async callable
+    A coroutine function taking one positional argument, the ``Datasette`` instance. Core calls ``await func(datasette)``.
+
+``name`` - string, optional
+    A name for the task, used to identify it in the ``/-/tasks`` introspection endpoint (:ref:`JsonDataView_tasks`) and in log messages. Defaults to ``func.__qualname__``. If the resulting name collides with an already-registered task, a ``-2``, ``-3``, ... suffix is appended.
+
+Registers supervised background work and returns a :ref:`BackgroundTask <BackgroundTask>` handle. Tasks registered during startup launch after all startup hooks finish; tasks registered after launch start immediately.
+
+See :ref:`internals_background_tasks` for examples, launch behavior, task supervision and cancellation.
+
+.. _datasette_start_background_tasks:
+
+await .start_background_tasks()
+-------------------------------
+
+Runs startup (if it has not already run) and launches every task registered with :ref:`datasette_add_background_task`.
+
+See :ref:`internals_background_tasks` for when tasks launch automatically, and :ref:`internals_background_tasks_explicit` for examples and startup considerations in tests and headless programs.
 
 .. _datasette_track_event:
 
@@ -1594,32 +1618,32 @@ datasette.client
 
 Plugins can make internal simulated HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins, while avoiding the overhead of making an external HTTP call to access those APIs.
 
-The ``datasette.client`` object is a wrapper around the `HTTPX Python library <https://www.python-httpx.org/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
+The ``datasette.client`` object is a wrapper around the `HTTPX2 Python library <https://httpx2.pydantic.dev/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
 
 It offers the following methods:
 
-``await datasette.client.get(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.get(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal GET request against that path.
 
-``await datasette.client.post(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.post(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal POST request. Use ``data={"name": "value"}`` to pass form parameters.
 
-``await datasette.client.options(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.options(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal OPTIONS request.
 
-``await datasette.client.head(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.head(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal HEAD request.
 
-``await datasette.client.put(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.put(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal PUT request.
 
-``await datasette.client.patch(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.patch(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal PATCH request.
 
-``await datasette.client.delete(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.delete(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal DELETE request.
 
-``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal request with the given HTTP method against that path.
 
 These methods can be used with :ref:`internals_datasette_urls` - for example:
@@ -1636,7 +1660,7 @@ These methods can be used with :ref:`internals_datasette_urls` - for example:
 
 ``datasette.client`` methods automatically take the current :ref:`setting_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path.
 
-For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation <https://www.python-httpx.org/async/>`__.
+For documentation on available ``**kwargs`` options and the shape of the HTTPX2 Response object refer to the `HTTPX2 Async documentation <https://httpx2.pydantic.dev/async/>`__.
 
 .. _internals_datasette_client_actor:
 
@@ -1765,6 +1789,135 @@ Use the ``format="json"`` (or ``"csv"`` or other formats supported by plugins) a
 
 These methods each return a ``datasette.utils.PrefixedUrlString`` object, which is a subclass of the Python ``str`` type. This allows the logic that considers the ``base_url`` setting to detect if that prefix has already been applied to the path.
 
+.. _datasette_lifecycle:
+
+Application lifecycle
+=====================
+
+Datasette guarantees a fixed sequence of events between the moment a ``Datasette`` instance is constructed and the moment its resources are released:
+
+1. ``Datasette(...)`` — the constructor runs synchronously and does not run plugin hooks.
+2. **Startup** — ``await datasette.invoke_startup()`` runs once: it populates the internal database's catalog of table schemas (:ref:`internals_internal`), loads canned queries and column type configuration, then calls every registered :ref:`plugin_hook_startup` hook, in plugin registration order. When Datasette is being served, table-count precomputation for immutable databases runs immediately before this, as part of the same startup sequence.
+3. **Background-task launch** — once *every* ``startup`` hook has finished (not before), every task registered with :ref:`datasette_add_background_task` — by any plugin — is launched. A task registered by one plugin's ``startup`` hook can safely depend on state set up by another plugin's ``startup`` hook, because launch only happens after the whole round of hooks completes.
+4. **Serving** — the instance handles requests (or, for headless or CLI use, does whatever the embedding program does with it).
+5. **Shutdown** — triggered by the ASGI ``lifespan.shutdown`` event (Ctrl-C, ``SIGTERM``) or the end of a ``datasette serve`` process: every :ref:`plugin_hook_shutdown` hook runs first, while background tasks are still alive, so a plugin can tell its own task to wind down gracefully; every still-running background task is then cancelled and given a five-second grace period to actually stop; finally every database connection is released via :ref:`datasette_close`.
+
+.. admonition:: Startup hooks run on the event loop that serves requests
+
+   In every trigger path below, ``startup`` hooks run on the same ``asyncio`` event loop that goes on to accept connections. It is safe to create loop-bound primitives — ``asyncio.Lock``, ``asyncio.Queue``, ``asyncio.Event``, a raw ``asyncio.create_task()`` call — inside a ``startup`` hook, and to register long-lived background work with :ref:`datasette_add_background_task` there. This was not always true: older Datasette versions ran startup on a temporary event loop in the CLI that was closed before the server's own loop was created, which could silently kill anything scheduled on it.
+
+Three trigger paths
+-------------------
+
+- ``datasette serve`` (CLI) — startup and ``uvicorn.Server.serve()`` both run inside a single ``asyncio.run()`` call, so there is exactly one event loop for the whole life of the process.
+- **ASGI lifespan** — ``Datasette.app()`` wires startup and background-task launch into the ``on_startup`` list, and shutdown into the ``on_shutdown`` list, of an internal ``AsgiLifespan`` wrapper. A spec-compliant ASGI server (uvicorn, hypercorn, and others) sends the ``lifespan.startup`` message and waits for ``lifespan.startup.complete`` before delivering any ``http`` or ``websocket`` scope, so startup — including every plugin's own internal-database migrations — is guaranteed to have finished before any request reaches Datasette, including requests seen by plugin :ref:`asgi_wrapper <plugin_asgi_wrapper>` middleware. If a ``startup`` hook raises, ``AsgiLifespan`` sends ``lifespan.startup.failed`` with the exception message instead of hanging or crashing ambiguously, so the host can abort the boot cleanly.
+- **First-request fallback** — an internal ``AsgiRunOnFirstRequest`` wrapper runs the same startup work as a safety net for hosts that never send ASGI lifespan events at all: some ASGI mounts, a bare ``app()`` embedded inside another framework, and :ref:`datasette.client <internals_datasette_client>` / test clients, which drive requests directly over ``httpx2.ASGITransport`` without ever emitting ``lifespan.startup``. It runs startup exactly once, the first time any non-lifespan scope arrives, guarded by a lock so that concurrent early requests can't run it twice.
+
+All three paths call the same idempotent internal methods, so it is safe for more than one of them to fire — lifespan startup completing and then a first request arriving afterwards is a no-op the second time. A host that never sends lifespan events and never goes through the CLI degrades to first-request timing: startup runs on the first request instead of before it, exactly as Datasette always worked prior to this lifecycle guarantee. This is a deliberate fallback rather than a regression — see :ref:`internals_background_tasks` for how to opt out of launching background tasks (the ``--get`` CLI path) or drive startup and launch explicitly (tests, headless embedders).
+
+.. _internals_background_tasks:
+
+Background tasks
+================
+
+Datasette can supervise long-lived background work for plugins, such as polling for updates. Register work using :ref:`datasette_add_background_task` and use the returned :ref:`BackgroundTask <BackgroundTask>` handle to inspect or cancel it. See :ref:`datasette_lifecycle` for how background tasks fit into the application's startup and shutdown sequence.
+
+Registering tasks
+-----------------
+
+Use :ref:`datasette_add_background_task` to register an async callable, typically from a :ref:`plugin_hook_startup` hook. The callable takes one argument, the ``Datasette`` instance. Tasks can also be registered later, including from a request handler.
+
+Registration is separate from launch. Calling this from a ``startup`` hook — the common case — buffers the task; core launches every registered task once *all* ``startup`` hooks have completed, as described in :ref:`datasette_lifecycle`. Calling it after launch has already happened — for example from a request handler, to start a per-job task dynamically — starts the task immediately instead.
+
+.. code-block:: python
+
+    import asyncio
+    from datasette import hookimpl
+
+
+    async def poll_for_updates(datasette):
+        while True:
+            await do_one_poll(datasette)
+            await asyncio.sleep(60)
+
+
+    @hookimpl
+    def startup(datasette):
+        datasette.add_background_task(
+            poll_for_updates, name="my-plugin-poller"
+        )
+
+Core owns the task for the rest of the process's life:
+
+- **A strong reference is kept forever**, so the task can never be silently garbage collected the way an unreferenced ``asyncio.create_task()`` call can be.
+- **A crash is logged, not swallowed.** If the task's callable raises anything other than ``asyncio.CancelledError``, the exception (with its traceback) is logged to the ``datasette.background_tasks`` logger and recorded on the handle's ``.exception``, and the task's ``.state`` becomes ``crashed``.
+- **Cancellation is coordinated.** On shutdown, every task that is still running is cancelled and given a grace period to stop — see :ref:`datasette_lifecycle`.
+
+.. _internals_background_tasks_launch:
+
+Launch matrix
+-------------
+
+Whether registered tasks actually launch depends on how the instance is being run:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Trigger
+     - Launches registered tasks?
+   * - ASGI lifespan (real server deployments)
+     - Yes, after ``lifespan.startup`` completes
+   * - First-request fallback (lifespan-less hosts)
+     - Yes, on the first request — parity with the lifespan case
+   * - ``datasette serve --get``
+     - Never
+   * - Tests / headless embedders
+     - Only if you call :ref:`datasette_start_background_tasks` explicitly
+
+``datasette --get`` never launches background tasks, even though its one-shot request flows through the same first-request fallback as everything else: it sets an internal flag before making that request specifically to suppress the launch, since a one-shot CLI invocation has no server loop left running afterwards to keep any launched tasks alive.
+
+.. _internals_background_tasks_explicit:
+
+Starting tasks explicitly
+-------------------------
+
+Call :ref:`datasette_start_background_tasks` to run startup (if it has not already run) and launch every task registered with :ref:`datasette_add_background_task`. This is the explicit equivalent of what happens automatically via ASGI lifespan or the first-request fallback in a served deployment — the entry point for tests and headless embedders (a cron-style CLI command that wants supervised background work without running a server) that need background tasks without going through either of those paths.
+
+.. code-block:: python
+
+    datasette = Datasette(memory=True)
+    await datasette.start_background_tasks()
+
+.. _BackgroundTask:
+
+BackgroundTask objects
+----------------------
+
+:ref:`datasette_add_background_task` returns a ``BackgroundTask`` handle with the following attributes:
+
+``.name`` - string
+    The task's (unique) name.
+
+``.state`` - string
+    One of ``registered`` (added but not yet launched), ``running``, ``completed`` (returned cleanly), ``crashed`` (raised an exception) or ``cancelled``.
+
+``.task`` - ``asyncio.Task`` or ``None``
+    The underlying ``asyncio.Task``, once launched. ``None`` while still ``registered``.
+
+``.exception`` - ``BaseException`` or ``None``
+    The exception that crashed the task, if ``.state`` is ``crashed``.
+
+``.started_at`` - string or ``None``
+    ISO 8601 UTC timestamp of when the task was launched.
+
+``.function`` - string
+    The callable's dotted module and qualified name, for example ``my_plugin.jobs.poll_for_updates``.
+
+``.cancel()``
+    Cancel the task. If it has already launched, this cancels the underlying ``asyncio.Task`` — ``.state`` becomes ``cancelled`` once the cancellation is observed. If it has not launched yet, it is removed from the queue so it never runs.
+
+This is also the shape of each entry returned by the ``/-/tasks`` JSON introspection endpoint — see :ref:`JsonDataView_tasks`.
+
 .. _internals_permission_classes:
 
 Permission classes and utilities
@@ -2023,8 +2176,8 @@ Example usage:
 
 .. _database_execute_write:
 
-await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True)
---------------------------------------------------------------------------------------------------------------------------
+await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True, time_limit_ms=2000)
+----------------------------------------------------------------------------------------------------------------------------------------------
 
 SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received.
 
@@ -2063,6 +2216,13 @@ Each call to ``execute_write()`` will be executed inside a transaction. Pass
 ``transaction=False`` for statements such as ``VACUUM`` that cannot run inside
 a transaction.
 
+Write statements have a default time limit of 2,000ms. Pass a different value
+using ``time_limit_ms=`` or use ``time_limit_ms=None`` to allow the statement to
+run without a time limit.
+
+This write limit is independent of the ``sql_time_limit_ms`` setting used for
+read queries. Changing that setting does not change the default write limit.
+
 .. _database_execute_write_script:
 
 await db.execute_write_script(sql, block=True)
@@ -2153,7 +2313,25 @@ The value returned from ``await database.execute_write_fn(...)`` will be the ret
 
 If your function raises an exception that exception will be propagated up to the ``await`` line.
 
-By default your function will be executed inside a transaction. You can pass ``transaction=False`` to disable this behavior, though if you do that you should be careful to manually apply transactions - ideally using the ``with conn:`` pattern, or you may see ``OperationalError: database table is locked`` errors.
+By default Datasette manages the transaction. For nested transactions, use `sqlite_utils.Database(conn).atomic() <https://sqlite-utils.datasette.io/en/stable/python-api.html#grouping-changes-with-db-atomic>`__. Pass ``transaction=False`` to manage transactions yourself.
+
+For example, archive an article and record the change in an audit log:
+
+.. code-block:: python
+
+    import sqlite_utils
+
+
+    def archive_article(conn):
+        db = sqlite_utils.Database(conn)
+        with db.atomic():
+            db["articles"].update(1, {"archived": True})
+            db["audit_log"].insert(
+                {"article_id": 1, "action": "archive"}
+            )
+
+
+    await database.execute_write_fn(archive_article)
 
 If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed.
 
@@ -2334,7 +2512,20 @@ No token, cookie, or hidden form field is needed. Any ``<form method="POST">`` i
 Datasette's internal database
 =============================
 
-Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a ``--internal`` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances.
+Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown.
+
+To persist internal data across Datasette instances, use the ``--internal`` option to specify the path to a SQLite database:
+
+.. code-block:: bash
+
+    datasette mydatabase.db --internal internal.db
+
+You can also set the ``DATASETTE_INTERNAL`` environment variable to specify this path without passing ``--internal`` each time:
+
+.. code-block:: bash
+
+    export DATASETTE_INTERNAL=/path/to/internal.db
+    datasette mydatabase.db
 
 Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_views``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases.
 
@@ -2623,12 +2814,12 @@ This example uses trace to record the start, end and duration of any HTTP GET re
 .. code-block:: python
 
     from datasette.tracer import trace
-    import httpx
+    import httpx2
 
 
     async def fetch_url(url):
         with trace("fetch-url", url=url):
-            async with httpx.AsyncClient() as client:
+            async with httpx2.AsyncClient() as client:
                 return await client.get(url)
 
 .. _internals_tracer_trace_child_tasks:

docs/introspection.rst

@@ -9,7 +9,7 @@ Each of these pages can be viewed in your browser. Add ``.json`` to the URL to g
 
 JSON responses that return an object include an ``"ok": true`` key, consistent with the rest of the :ref:`JSON API <json_api>`.
 
-The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise <json_api_stability>`, with the exception of the debug endpoints ``/-/threads`` and ``/-/actions``, whose shapes may change in future releases.
+The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise <json_api_stability>`, with the exception of the debug endpoints ``/-/threads``, ``/-/tasks`` and ``/-/actions``, whose shapes may change in future releases.
 
 .. _JsonDataView_metadata:
 
@@ -76,7 +76,7 @@ Shows the version of Datasette, Python and SQLite. `Versions example <https://la
 /-/plugins
 ----------
 
-Shows a list of currently installed plugins and their versions. `Plugins example <https://san-francisco.datasettes.com/-/plugins>`_:
+Shows a list of currently installed plugins and their versions. `Plugins example <https://datasette.io/-/plugins>`_:
 
 .. code-block:: json
 
@@ -278,6 +278,42 @@ Shows details of threads and ``asyncio`` tasks. This endpoint requires the ``per
         ]
     }
 
+.. _JsonDataView_tasks:
+
+/-/tasks
+--------
+
+Shows the state of every supervised background task registered with :ref:`datasette.add_background_task() <datasette_add_background_task>`; see also :ref:`BackgroundTask <BackgroundTask>` for what each field below means, and :ref:`datasette_lifecycle` for when tasks are launched. This endpoint requires the ``permissions-debug`` permission, since a crashed task's ``exception`` field can reveal internals such as file paths or query text:
+
+.. code-block:: json
+
+    {
+        "ok": true,
+        "tasks": [
+            {
+                "name": "my_plugin.poll_for_updates",
+                "state": "running",
+                "function": "my_plugin.poll_for_updates",
+                "started_at": "2026-07-30T12:00:00+00:00",
+                "exception": null
+            },
+            {
+                "name": "my_plugin.broken_task",
+                "state": "crashed",
+                "function": "my_plugin.broken_task",
+                "started_at": "2026-07-30T12:00:00+00:00",
+                "exception": "ValueError('something went wrong')"
+            }
+        ],
+        "launched": true
+    }
+
+Each entry's ``function`` identifies the callable by its dotted module and qualified name.
+
+Each entry's ``state`` is one of ``registered`` (added but not yet launched), ``running``, ``completed``, ``crashed`` or ``cancelled``. ``exception`` is a one-line ``repr()`` of the exception for a ``crashed`` task, or ``null`` otherwise - the full traceback is written to the ``datasette.background_tasks`` logger instead, to keep this payload skimmable.
+
+The top-level ``launched`` flag reports whether the instance has run its one-time background task launch (after ``startup`` hooks finish, or via lifespan/first-request/:ref:`start_background_tasks() <datasette_start_background_tasks>`). It distinguishes "no tasks have been registered" (``tasks`` is empty either way) from "tasks are registered but nothing has armed the launch yet" (``launched`` is ``false`` and every task's ``state`` is still ``registered``) - useful when debugging a host that never triggers Datasette's lifespan events.
+
 .. _JsonDataView_actor:
 
 /-/actor

docs/javascript_plugins.rst

@@ -1,7 +1,7 @@
 .. _javascript_plugins:
 
-JavaScript plugins
-==================
+JavaScript in plugins
+=====================
 
 Datasette can run custom JavaScript in several different ways:
 
@@ -474,6 +474,158 @@ Custom fields are responsible for preserving the accessibility of the form:
 
 Plugins should not submit the row themselves from inside ``makeColumnField()`` controls. Datasette owns the insert/edit dialog lifecycle, form submission, API call, error handling and row refresh.
 
+.. _javascript_plugins_modals:
+
+Reusable modal dialogs
+----------------------
+
+Plugins can use ``DatasetteModal`` to create dialogs with the same appearance and keyboard behavior as Datasette's built-in dialogs. The component provides a native modal dialog, shared styles, Escape and backdrop dismissal, busy-state dismissal guards and focus restoration.
+
+Creating a dialog
+~~~~~~~~~~~~~~~~~
+
+``DatasetteModal.create()`` returns a detached ``<datasette-modal>`` element containing a native ``<dialog>``. Access that native element through ``modal.dialog``. Populate its content before appending the wrapper to the page, then call ``modal.show()`` to open it.
+
+This example uses the :ref:`datasette_init event <javascript_datasette_init>` to add a button that opens a dialog:
+
+.. code-block:: javascript
+
+    document.addEventListener("datasette_init", () => {
+        const openButton = document.createElement("button");
+        openButton.type = "button";
+        openButton.textContent = "Open example dialog";
+        // Indicate that this button opens a dialog:
+        openButton.setAttribute("aria-haspopup", "dialog");
+        // Identify which dialog it controls:
+        openButton.setAttribute("aria-controls", "my-plugin-dialog");
+
+        const modal = DatasetteModal.create();
+        const dialog = modal.dialog;
+        dialog.id = "my-plugin-dialog";
+        // Tell screenreaders the dialog is labelled by #my-plugin-dialog-title
+        dialog.setAttribute("aria-labelledby", "my-plugin-dialog-title");
+        dialog.innerHTML = `
+          <div class="modal-header">
+            <h2 class="modal-title" id="my-plugin-dialog-title">
+              Example dialog
+            </h2>
+          </div>
+          <div class="modal-body">
+            This dialog uses Datasette's shared styles and keyboard behavior.
+          </div>
+          <div class="modal-footer">
+            <button type="button" class="modal-btn modal-btn-ghost">Close</button>
+          </div>`;
+
+        const closeButton = dialog.querySelector("button");
+        closeButton.addEventListener("click", () => {
+            modal.requestClose("cancel");
+        });
+        openButton.addEventListener("click", () => {
+            modal.show({ returnFocusTo: openButton, initialFocus: closeButton });
+        });
+
+        document.body.append(modal);
+        document.querySelector("section.content").append(openButton);
+    });
+
+Opening and closing
+~~~~~~~~~~~~~~~~~~~
+
+``modal.show(options)``
+    Opens the native dialog using ``showModal()``. ``options`` is an optional object with these optional properties:
+
+    - ``returnFocusTo`` (DOM element): Focus returns to this element when the dialog closes. Defaults to the element with keyboard focus immediately before the dialog opens.
+    - ``initialFocus`` (DOM element or function): An element inside the dialog whose ``focus()`` method will be called, or a function called with no arguments that moves focus itself.
+
+    Use this to focus on an input field when the dialog opens.
+
+``modal.close(options)``
+    Closes the dialog directly. ``options`` is an optional object with one optional property:
+
+    - ``restoreFocus`` (boolean): Whether closing returns focus to the element recorded by ``show()``. Defaults to ``true``.
+
+``modal.requestClose(source)``
+    Alternative to ``.close()`` that requests dismissal through the busy-state and ``beforeClose`` guards described below. Returns ``true`` if it closes the dialog, or ``false`` if the dialog is already closed or a guard prevents dismissal. Close and Cancel buttons should use this method.
+
+    ``source`` is an optional string that is passed to ``beforeClose`` and identifies what requested dismissal. Datasette supplies ``"escape"`` for the Escape key or a native cancel event and ``"backdrop"`` for a click outside the dialog. ``source`` defaults to ``"cancel"``.
+
+Listen for the native dialog's ``close`` event to clean up resources such as pending requests or custom fields:
+
+.. code-block:: javascript
+
+    modal.dialog.addEventListener("close", () => {
+        // Clean up content-specific resources here.
+    });
+
+If the dialog is no longer needed, remove the wrapper with ``modal.remove()``.
+
+Dismissal guards and busy state
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can set ``modal.beforeClose`` to a synchronous function that receives the ``source`` string described above and returns ``false`` in order to keep the dialog open.
+
+Use ``source`` to decide what to do. This example prompts the user to ask if they want to discard unsaved changes - for example if they click outside the modal or hit Escape - but doesn't prompt them if they clicked a button like the Close one above that sets the ``source`` string to ``cancel``.
+
+.. code-block:: javascript
+
+    modal.beforeClose = (source) => {
+        if (source === "cancel") return true;
+        return confirm("Discard unsaved changes?");
+    };
+
+
+Set ``modal.busy = true`` while saving to prevent user dismissal.  While busy, ``requestClose()`` returns ``false`` without calling ``beforeClose``.
+
+If an operation fails, set ``modal.busy = false`` so the user can retry or close the dialog. A successful operation can call ``modal.close()`` even while busy.
+
+.. _javascript_plugins_modal_classes:
+
+Shared CSS classes
+~~~~~~~~~~~~~~~~~~
+
+The classes in the example above provide built-in styling. The following classes can be used by your modal:
+
+``datasette-modal``
+    Added automatically to the native ``<dialog>`` when the wrapper is connected to the page. Provides the dialog's sizing, background, rounded corners, shadow, backdrop and animations.
+
+``modal-header``
+    Adds padding, a bottom border and a horizontal layout for the title and optional metadata.
+
+``modal-title``
+    Sets the title's font size, weight and color. Use ``aria-labelledby`` to associate the title with the dialog.
+
+``modal-meta``
+    Styles optional metadata, such as a selected-item count, as small monospace text with a rounded background.
+
+``modal-body``
+    Adds padding and makes overflowing content scroll while the header and footer remain visible. Sets ``min-height: 0``, ``overflow: auto`` and ``padding: 16px 24px 24px``.
+
+``modal-footer``
+    Adds padding, a top border and a background to the action area. Arranges its contents horizontally, with buttons aligned to the right.
+
+``footer-info``
+    Styles supporting text in the footer and lets it fill the space before the action buttons.
+
+``modal-btn``
+    Provides base button styling, including padding, rounded corners, font and disabled appearance. Use it together with ``modal-btn-primary`` or ``modal-btn-ghost``.
+
+``modal-btn-primary``
+    Gives a button an accent-colored background and white text, suitable for a primary action such as Save.
+
+``modal-btn-ghost``
+    Gives a button a transparent background, muted text and a border, suitable for a secondary action such as Close or Cancel.
+
+These button classes are also used by Datasette's built-in dialogs.
+
+You can customize layout and sizing without adding extra classes. For example, this CSS uses the dialog's existing ID to widen it while keeping it inside the viewport:
+
+.. code-block:: css
+
+    dialog#my-plugin-dialog {
+        width: min(720px, calc(100vw - 32px));
+    }
+
 .. _javascript_datasette_manager_selectors:
 
 Selectors

docs/json_api.rst

@@ -48,6 +48,7 @@ Some JSON endpoints are **exempt** from this promise:
   debug playground.
 - Debug and support endpoints are documented so you can use them, but their
   JSON shapes are not frozen: :ref:`/-/threads <JsonDataView_threads>`,
+  :ref:`/-/tasks <JsonDataView_tasks>`,
   :ref:`/-/actions <JsonDataView_actions>`,
   the :ref:`permission debug endpoints <PermissionsDebugView>`
   (``/-/allowed``, ``/-/rules``, ``/-/check``) and the
@@ -1326,6 +1327,23 @@ The following extras are available for arbitrary SQL query responses and stored,
 
 .. [[[end]]]
 
+.. _TableCountView:
+
+Counting all matching rows
+--------------------------
+
+``POST /<database>/<table>/-/count`` returns an exact count of the rows matching the table's query string filters::
+
+    POST /fixtures/facetable/-/count?state=CA
+
+    {"ok": true, "count": 10}
+
+The endpoint supports the same column, search and plugin filters as the table page. Pagination and display options such as ``_next``, ``_size`` and ``_sort`` do not affect the count.
+
+This requires ``view-table`` permission. ``execute-sql`` permission is only needed if using ``_where`` filters.
+
+Unlike the ``count`` extra, this count is not capped by the row count limit. The usual SQL time limit still applies; a timed-out count returns a 400 JSON error.
+
 .. _TableAutocompleteView:
 
 Table autocomplete
@@ -1661,6 +1679,8 @@ The request body is always parsed as JSON, regardless of the request's ``Content
 
 The row-based write APIs can write :ref:`binary values in JSON <binary_json_format>` using Datasette's Base64 representation for BLOB data.
 
+Structured inserts, upserts, updates and deletes only support ordinary SQLite tables. Virtual tables and their internal shadow tables are rejected, including when adding rows to an existing table through the create-table API. Writes to ordinary content tables can still update full-text search indexes through configured triggers.
+
 .. _ExecuteWriteView:
 
 Executing write SQL

docs/plugin_hooks.rst

@@ -217,7 +217,7 @@ Extra template variables that should be made available in the rendered template
 ``datasette`` - :ref:`internals_datasette`
     You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
 
-This hook can return one of three different types:
+This hook supports the following return values:
 
 Dictionary
     If you return a dictionary its keys and values will be merged into the template context.
@@ -228,6 +228,9 @@ Function that returns a dictionary
 Function that returns an awaitable function that returns a dictionary
     You can also return a function which returns an awaitable function which returns a dictionary.
 
+``None``
+    The hook itself, or a function or awaitable it returns, can return ``None`` when no extra variables are needed. Variables returned by other plugins are still included.
+
 Datasette runs Jinja2 in `async mode <https://jinja.palletsprojects.com/en/2.10.x/api/#async-support>`__, which means you can add awaitable functions to the template scope and they will be automatically awaited when they are rendered by the template.
 
 .. warning::
@@ -254,8 +257,6 @@ This example returns an awaitable function which adds a list of ``hidden_table_n
                 return {
                     "hidden_table_names": await db.hidden_table_names()
                 }
-            else:
-                return {}
 
         return hidden_table_names
 
@@ -495,10 +496,10 @@ Lets you customize the display of values within table cells in the HTML table vi
     The name of the column being rendered
 
 ``table`` - string or None
-    The name of the table - or ``None`` if this is a custom SQL query
+    The name of the table or view - or ``None`` if this is a custom SQL query
 
 ``pks`` - list of strings
-    The primary key column names for the table being rendered. For tables without an explicitly defined primary key, this will be ``["rowid"]``. For custom SQL queries and views (where ``table`` is ``None``), this will be an empty list ``[]``.
+    The primary key column names for the table being rendered. For tables without an explicitly defined primary key, this will be ``["rowid"]``. For custom SQL queries and views, this will be an empty list ``[]``.
 
 ``database`` - string
     The name of the database
@@ -1107,7 +1108,7 @@ Return an `ASGI <https://asgi.readthedocs.io/>`__ middleware wrapper function th
 
 This is a very powerful hook. You can use it to manipulate the entire Datasette response, or even to configure new URL routes that will be handled by your own custom code.
 
-You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette <https://www.starlette.io/middleware/>`__.
+You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette <https://starlette.dev/middleware/>`__.
 
 This example plugin adds a ``x-databases`` HTTP header listing the currently attached databases:
 
@@ -1157,7 +1158,7 @@ Examples: `datasette-cors <https://datasette.io/plugins/datasette-cors>`__, `dat
 startup(datasette)
 ------------------
 
-This hook fires when the Datasette application server first starts up.
+This hook fires when the Datasette application server first starts up. It runs on the same event loop that goes on to serve requests, so it is safe to create loop-bound primitives and register background work here — see :ref:`datasette_lifecycle` for the full guarantee and the three ways startup can be triggered.
 
 Here is an example that validates required plugin configuration. The server will fail to start and show an error if the validation check fails:
 
@@ -1195,6 +1196,7 @@ Potential use-cases:
 * Create database tables that a plugin needs on startup
 * Validate the configuration for a plugin on startup, and raise an error if it is invalid
 * Raise a ``datasette.utils.StartupError("message")`` exception to prevent Datasette from starting and display that message to the user.
+* Register supervised long-lived background work using :ref:`datasette_add_background_task`, which core launches once every plugin's ``startup()`` hook has finished.
 
 .. note::
 
@@ -1211,6 +1213,31 @@ Potential use-cases:
 
 Examples: `datasette-saved-queries <https://datasette.io/plugins/datasette-saved-queries>`__, `datasette-init <https://datasette.io/plugins/datasette-init>`__
 
+.. _plugin_hook_shutdown:
+
+shutdown(datasette)
+-------------------
+
+This hook fires once, when the Datasette application server is shutting down gracefully - triggered by the ASGI ``lifespan.shutdown`` event, which includes pressing Ctrl-C or sending ``SIGTERM`` to a ``datasette serve`` process. It is not called on a hard kill (``SIGKILL``), since there is no opportunity to run any code in that case.
+
+Like ``startup()``, this can be a regular function or it can return an async function to be awaited.
+
+It runs before Datasette cancels any background tasks it is supervising (see :ref:`datasette_add_background_task`) and before it closes its database connections, so you can use it to tell your plugin's own background work to stop gracefully while a database connection is still available to write out any final state. See :ref:`datasette_lifecycle` for exactly where this fits into the full startup-to-shutdown sequence:
+
+.. code-block:: python
+
+    @hookimpl
+    def shutdown(datasette):
+        async def inner():
+            db = datasette.get_database()
+            await db.execute_write(
+                "insert into shutdown_log (at) values (datetime('now'))"
+            )
+
+        return inner
+
+If your ``shutdown()`` hook raises an exception it will be logged but not re-raised, so one plugin's broken shutdown code cannot prevent other plugins - or Datasette itself - from finishing their own teardown.
+
 .. _plugin_hook_actor_from_request:
 
 actor_from_request(datasette, request)

docs/plugins.rst

@@ -261,6 +261,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship
                 "permission_resources_sql"
             ]
         },
+        {
+            "name": "datasette.default_permissions.sqlite_statistics",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "permission_resources_sql"
+            ]
+        },
         {
             "name": "datasette.default_permissions.tokens",
             "static": false,

docs/settings.rst

@@ -67,10 +67,21 @@ The following options can be set using ``--setting name value``, or by storing t
 default_allow_sql
 ~~~~~~~~~~~~~~~~~
 
+.. [[[cog
+    from settings_doc import setting_default
+    setting_default(cog, "default_allow_sql")
+.. ]]]
+
+Default: ``on``
+
+.. [[[end]]]
+
 Should users be able to execute arbitrary SQL queries by default?
 
 Setting this to ``off`` causes permission checks for :ref:`actions_execute_sql` to fail by default.
 
+This setting controls the ability to submit arbitrary SQL. It does not disable structured table-browsing features that use SQL generated by Datasette, such as sorting, column filters and :ref:`facets`. Use :ref:`setting_allow_facet` to control whether users can request facets.
+
 ::
 
     datasette mydatabase.db --setting default_allow_sql off
@@ -82,6 +93,14 @@ Another way to achieve this is to add ``"allow_sql": false`` to your ``datasette
 default_page_size
 ~~~~~~~~~~~~~~~~~
 
+.. [[[cog
+    setting_default(cog, "default_page_size")
+.. ]]]
+
+Default: ``100``
+
+.. [[[end]]]
+
 The default number of rows returned by the table page. You can over-ride this on a per-page basis using the ``?_size=80`` query string parameter, provided you do not specify a value higher than the ``max_returned_rows`` setting. You can set this default using ``--setting`` like so::
 
     datasette mydatabase.db --setting default_page_size 50
@@ -91,7 +110,15 @@ The default number of rows returned by the table page. You can over-ride this on
 sql_time_limit_ms
 ~~~~~~~~~~~~~~~~~
 
-By default, queries have a time limit of one second. If a query takes longer than this to run Datasette will terminate the query and return an error.
+.. [[[cog
+    setting_default(cog, "sql_time_limit_ms")
+.. ]]]
+
+Default: ``1000``
+
+.. [[[end]]]
+
+Time limit for SQL queries, in milliseconds. If a query takes longer than this to run Datasette will terminate the query and return an error.
 
 If this time limit is too short for you, you can customize it using the ``sql_time_limit_ms`` limit - for example, to increase it to 3.5 seconds::
 
@@ -108,7 +135,15 @@ This would set the time limit to 100ms for that specific query. This feature is
 max_returned_rows
 ~~~~~~~~~~~~~~~~~
 
-Datasette returns a maximum of 1,000 rows of data at a time. If you execute a query that returns more than 1,000 rows, Datasette will return the first 1,000 and include a warning that the result set has been truncated. You can use OFFSET/LIMIT or other methods in your SQL to implement pagination if you need to return more than 1,000 rows.
+.. [[[cog
+    setting_default(cog, "max_returned_rows")
+.. ]]]
+
+Default: ``1000``
+
+.. [[[end]]]
+
+The maximum number of rows Datasette returns at a time. If you execute a query that exceeds this limit, Datasette will truncate the result set and include a warning. You can use OFFSET/LIMIT or other methods in your SQL to implement pagination if you need to return more rows.
 
 You can increase or decrease this limit like so::
 
@@ -119,7 +154,15 @@ You can increase or decrease this limit like so::
 max_insert_rows
 ~~~~~~~~~~~~~~~
 
-Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`. Defaults to 100.
+.. [[[cog
+    setting_default(cog, "max_insert_rows")
+.. ]]]
+
+Default: ``100``
+
+.. [[[end]]]
+
+Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`.
 
 You can increase or decrease this limit like so::
 
@@ -130,7 +173,15 @@ You can increase or decrease this limit like so::
 max_post_body_bytes
 ~~~~~~~~~~~~~~~~~~~
 
-Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API <json_api_write>`. Requests with larger bodies are rejected with an HTTP 413 error. Defaults to 2,097,152 (2MB).
+.. [[[cog
+    setting_default(cog, "max_post_body_bytes")
+.. ]]]
+
+Default: ``2097152``
+
+.. [[[end]]]
+
+Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API <json_api_write>`. Requests with larger bodies are rejected with an HTTP 413 error.
 
 This limit exists to protect against memory exhaustion: unlike file uploads handled by ``request.form()``, which stream to disk, these bodies are held entirely in memory and parsing them as JSON can multiply their memory footprint several times over.
 
@@ -147,7 +198,15 @@ Set it to 0 to disable the limit entirely::
 num_sql_threads
 ~~~~~~~~~~~~~~~
 
-Maximum number of threads in the thread pool Datasette uses to execute SQLite queries. Defaults to 3.
+.. [[[cog
+    setting_default(cog, "num_sql_threads")
+.. ]]]
+
+Default: ``3``
+
+.. [[[end]]]
+
+Maximum number of threads in the thread pool Datasette uses to execute SQLite queries.
 
 ::
 
@@ -160,9 +219,17 @@ Setting this to 0 turns off threaded SQL queries entirely - useful for environme
 allow_facet
 ~~~~~~~~~~~
 
+.. [[[cog
+    setting_default(cog, "allow_facet")
+.. ]]]
+
+Default: ``on``
+
+.. [[[end]]]
+
 Allow users to specify columns they would like to facet on using the ``?_facet=COLNAME`` URL parameter to the table view.
 
-This is enabled by default. If disabled, facets will still be displayed if they have been specifically enabled in ``metadata.json`` configuration for the table.
+If disabled, facets will still be displayed if they have been specifically enabled in ``metadata.json`` configuration for the table.
 
 Here's how to disable this feature::
 
@@ -173,7 +240,15 @@ Here's how to disable this feature::
 default_facet_size
 ~~~~~~~~~~~~~~~~~~
 
-The default number of unique rows returned by :ref:`facets` is 30. You can customize it like this::
+.. [[[cog
+    setting_default(cog, "default_facet_size")
+.. ]]]
+
+Default: ``30``
+
+.. [[[end]]]
+
+The default number of unique rows returned by :ref:`facets`. You can customize it like this::
 
     datasette mydatabase.db --setting default_facet_size 50
 
@@ -182,7 +257,15 @@ The default number of unique rows returned by :ref:`facets` is 30. You can custo
 facet_time_limit_ms
 ~~~~~~~~~~~~~~~~~~~
 
-This is the time limit Datasette allows for calculating a facet, which defaults to 200ms::
+.. [[[cog
+    setting_default(cog, "facet_time_limit_ms")
+.. ]]]
+
+Default: ``200``
+
+.. [[[end]]]
+
+The time limit in milliseconds Datasette allows for calculating a facet. You can customize it like this::
 
     datasette mydatabase.db --setting facet_time_limit_ms 1000
 
@@ -191,7 +274,15 @@ This is the time limit Datasette allows for calculating a facet, which defaults
 facet_suggest_time_limit_ms
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-When Datasette calculates suggested facets it needs to run a SQL query for every column in your table. The default for this time limit is 50ms to account for the fact that it needs to run once for every column. If the time limit is exceeded the column will not be suggested as a facet.
+.. [[[cog
+    setting_default(cog, "facet_suggest_time_limit_ms")
+.. ]]]
+
+Default: ``50``
+
+.. [[[end]]]
+
+When Datasette calculates suggested facets it needs to run a SQL query for every column in your table. This time limit, in milliseconds, applies separately to each query. If the time limit is exceeded the column will not be suggested as a facet.
 
 You can increase this time limit like so::
 
@@ -202,7 +293,15 @@ You can increase this time limit like so::
 suggest_facets
 ~~~~~~~~~~~~~~
 
-Should Datasette calculate suggested facets? On by default, turn this off like so::
+.. [[[cog
+    setting_default(cog, "suggest_facets")
+.. ]]]
+
+Default: ``on``
+
+.. [[[end]]]
+
+Should Datasette calculate suggested facets? Turn this off like so::
 
     datasette mydatabase.db --setting suggest_facets off
 
@@ -211,7 +310,15 @@ Should Datasette calculate suggested facets? On by default, turn this off like s
 allow_download
 ~~~~~~~~~~~~~~
 
-Should users be able to download the original SQLite database using a link on the database index page? This is turned on by default. However, databases can only be downloaded if they are served in immutable mode and not in-memory. If downloading is unavailable for either of these reasons, the download link is hidden even if ``allow_download`` is on. To disable database downloads, use the following::
+.. [[[cog
+    setting_default(cog, "allow_download")
+.. ]]]
+
+Default: ``on``
+
+.. [[[end]]]
+
+Should users be able to download the original SQLite database using a link on the database index page? Databases can only be downloaded if they are served in immutable mode and not in-memory. If downloading is unavailable for either of these reasons, the download link is hidden even if ``allow_download`` is on. To disable database downloads, use the following::
 
     datasette mydatabase.db --setting allow_download off
 
@@ -220,9 +327,17 @@ Should users be able to download the original SQLite database using a link on th
 allow_signed_tokens
 ~~~~~~~~~~~~~~~~~~~
 
+.. [[[cog
+    setting_default(cog, "allow_signed_tokens")
+.. ]]]
+
+Default: ``on``
+
+.. [[[end]]]
+
 Should users be able to create signed API tokens to access Datasette?
 
-This is turned on by default. Use the following to turn it off::
+Use the following to turn it off::
 
     datasette mydatabase.db --setting allow_signed_tokens off
 
@@ -233,9 +348,17 @@ Turning this setting off will disable the ``/-/create-token`` page, :ref:`descri
 max_signed_tokens_ttl
 ~~~~~~~~~~~~~~~~~~~~~
 
+.. [[[cog
+    setting_default(cog, "max_signed_tokens_ttl")
+.. ]]]
+
+Default: ``0``
+
+.. [[[end]]]
+
 Maximum allowed expiry time for signed API tokens created by users.
 
-Defaults to ``0`` which means no limit - tokens can be created that will never expire.
+A value of ``0`` means no limit - tokens can be created that will never expire.
 
 Set this to a value in seconds to limit the maximum expiry time. For example, to set that limit to 24 hours you would use::
 
@@ -248,18 +371,36 @@ This setting is enforced when incoming tokens are processed.
 default_cache_ttl
 ~~~~~~~~~~~~~~~~~
 
-Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-age=X``. Can be over-ridden on a per-request basis using the ``?_ttl=`` query string parameter. Set this to ``0`` to disable HTTP caching entirely. Defaults to 5 seconds.
+.. [[[cog
+    setting_default(cog, "default_cache_ttl")
+.. ]]]
+
+Default: ``5``
+
+.. [[[end]]]
+
+Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-age=X``. Can be over-ridden on a per-request basis using the ``?_ttl=`` query string parameter. Set this to ``0`` to disable HTTP caching entirely.
 
 ::
 
     datasette mydatabase.db --setting default_cache_ttl 60
 
+Dynamic responses for authenticated actors, requests with cookies or an ``Authorization`` header, and responses that set cookies use ``Cache-Control: private, no-store``. This takes precedence over ``default_cache_ttl`` and ``?_ttl=``, even when cache headers are otherwise disabled. Anonymous dynamic responses vary by ``Cookie`` and ``Authorization``. Static assets retain their own cache policy.
+
 .. _setting_cache_size_kb:
 
 cache_size_kb
 ~~~~~~~~~~~~~
 
-Sets the amount of memory SQLite uses for its `per-connection cache <https://www.sqlite.org/pragma.html#pragma_cache_size>`_, in KB.
+.. [[[cog
+    setting_default(cog, "cache_size_kb")
+.. ]]]
+
+Default: ``0``
+
+.. [[[end]]]
+
+Sets the amount of memory SQLite uses for its `per-connection cache <https://www.sqlite.org/pragma.html#pragma_cache_size>`_, in KB. Set this to ``0`` to use SQLite's default cache size.
 
 ::
 
@@ -270,9 +411,17 @@ Sets the amount of memory SQLite uses for its `per-connection cache <https://www
 allow_csv_stream
 ~~~~~~~~~~~~~~~~
 
+.. [[[cog
+    setting_default(cog, "allow_csv_stream")
+.. ]]]
+
+Default: ``on``
+
+.. [[[end]]]
+
 Enables :ref:`the CSV export feature <csv_export>` where an entire table
 (potentially hundreds of thousands of rows) can be exported as a single CSV
-file. This is turned on by default - you can turn it off like this:
+file. You can turn it off like this:
 
 ::
 
@@ -283,8 +432,16 @@ file. This is turned on by default - you can turn it off like this:
 max_csv_mb
 ~~~~~~~~~~
 
-The maximum size of CSV that can be exported, in megabytes. Defaults to 100MB.
-You can disable the limit entirely by settings this to 0:
+.. [[[cog
+    setting_default(cog, "max_csv_mb")
+.. ]]]
+
+Default: ``100``
+
+.. [[[end]]]
+
+The maximum size of CSV that can be exported, in megabytes.
+You can disable the limit entirely by setting this to 0:
 
 ::
 
@@ -295,6 +452,14 @@ You can disable the limit entirely by settings this to 0:
 truncate_cells_html
 ~~~~~~~~~~~~~~~~~~~
 
+.. [[[cog
+    setting_default(cog, "truncate_cells_html")
+.. ]]]
+
+Default: ``2048``
+
+.. [[[end]]]
+
 In the HTML table view, truncate any strings that are longer than this value.
 The full value will still be available in CSV, JSON and on the individual row
 HTML page. Set this to 0 to disable truncation.
@@ -308,6 +473,14 @@ HTML page. Set this to 0 to disable truncation.
 force_https_urls
 ~~~~~~~~~~~~~~~~
 
+.. [[[cog
+    setting_default(cog, "force_https_urls")
+.. ]]]
+
+Default: ``off``
+
+.. [[[end]]]
+
 Forces self-referential URLs in the JSON output to always use the ``https://``
 protocol. This is useful for cases where the application itself is hosted using
 HTTP but is served to the outside world via a proxy that enables HTTPS.
@@ -321,6 +494,14 @@ HTTP but is served to the outside world via a proxy that enables HTTPS.
 template_debug
 ~~~~~~~~~~~~~~
 
+.. [[[cog
+    setting_default(cog, "template_debug")
+.. ]]]
+
+Default: ``off``
+
+.. [[[end]]]
+
 This setting enables template context debug mode, which is useful to help understand what variables are available to custom templates when you are writing them.
 
 Enable it like this::
@@ -340,6 +521,14 @@ Some examples:
 trace_debug
 ~~~~~~~~~~~
 
+.. [[[cog
+    setting_default(cog, "trace_debug")
+.. ]]]
+
+Default: ``off``
+
+.. [[[end]]]
+
 This setting enables appending ``?_trace=1`` to any page in order to see the SQL queries and other trace information that was used to generate that page.
 
 Enable it like this::
@@ -358,6 +547,14 @@ See :ref:`internals_tracer` for details on how to hook into this mechanism as a
 base_url
 ~~~~~~~~
 
+.. [[[cog
+    setting_default(cog, "base_url")
+.. ]]]
+
+Default: ``/``
+
+.. [[[end]]]
+
 If you are running Datasette behind a proxy, it may be useful to change the root path used for the Datasette instance.
 
 For example, if you are sending traffic from ``https://www.example.com/tools/datasette/`` through to a proxied Datasette instance you may wish Datasette to use ``/tools/datasette/`` as its root URL.

docs/settings_doc.py

@@ -0,0 +1,10 @@
+"""Cog helper for documenting setting defaults from Datasette's registry."""
+
+
+def setting_default(cog, name):
+    from datasette.app import DEFAULT_SETTINGS
+
+    default = DEFAULT_SETTINGS[name]
+    if isinstance(default, bool):
+        default = "on" if default else "off"
+    cog.out(f"\nDefault: ``{default}``\n\n")

docs/testing_plugins.rst

@@ -25,7 +25,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl
         )
 
 
-This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX <https://www.python-httpx.org/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
+This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX2 <https://httpx2.pydantic.dev/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
 
 This test also uses the `pytest-asyncio <https://pypi.org/project/pytest-asyncio/>`__ package to add support for ``async def`` test functions running under pytest.
 
@@ -78,9 +78,19 @@ Creating a ``Datasette()`` instance like this as useful shortcut in tests, but t
     datasette = Datasette(memory=True)
     await datasette.invoke_startup()
 
-This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls.
+This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls. It runs on the same event loop that runs your test, matching the guarantee described in :ref:`datasette_lifecycle`.
 
-If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request.
+If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request, via the first-request fallback described in :ref:`datasette_lifecycle`.
+
+If your plugin also registers work with :ref:`datasette_add_background_task` (typically from a ``startup`` hook) and your test needs that work to actually run, call ``await datasette.start_background_tasks()`` as well - ``invoke_startup()`` alone only runs ``startup`` hooks, it does not launch anything they registered:
+
+.. code-block:: python
+
+    datasette = Datasette(memory=True)
+    await datasette.start_background_tasks()
+    # Any tasks registered by a startup() hook are now running
+
+A request made through ``datasette.client`` arms both startup and background-task launch automatically, since they're both part of the same first-request fallback - ``start_background_tasks()`` is for tests that need tasks running without making an HTTP request first.
 
 .. _testing_plugins_datasette_fixtures_database:
 
@@ -154,7 +164,7 @@ If you need to opt out of this behavior, add the following to your ``pytest.ini`
 Using datasette.client in tests
 -------------------------------
 
-The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client <https://www.python-httpx.org/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
+The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX2 async client <https://httpx2.pydantic.dev/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
 
 A simple test looks like this:
 
@@ -273,22 +283,22 @@ If you want to create that test database repeatedly for every individual test fu
 
 .. _testing_plugins_pytest_httpx:
 
-Testing outbound HTTP calls with pytest-httpx
----------------------------------------------
+Testing outbound HTTP calls with pytest-httpx2
+----------------------------------------------
 
 If your plugin makes outbound HTTP calls - for example datasette-auth-github or datasette-import-table - you may need to mock those HTTP requests in your tests.
 
-The `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__ package is a useful library for mocking calls. It can be tricky to use with Datasette though since it mocks all HTTPX requests, and Datasette's own testing mechanism uses HTTPX internally.
+The `pytest-httpx2 <https://pypi.org/project/pytest-httpx2/>`__ package provides a ``httpx2_mock`` fixture, built on `respx <https://lundberg.github.io/respx/>`__, for mocking outbound calls made using HTTPX2.
 
-To avoid breaking your tests, you can return ``["localhost"]`` from the ``non_mocked_hosts()`` fixture.
+Datasette's own ``datasette.client`` mechanism uses HTTPX2 internally too, but those requests are passed directly to the ASGI application rather than being sent over the network, so they are not affected by the mock.
 
-As an example, here's a very simple plugin which executes an HTTP response and returns the resulting content:
+As an example, here's a very simple plugin which executes an HTTP request and returns the resulting content:
 
 .. code-block:: python
 
     from datasette import hookimpl
     from datasette.utils.asgi import Response
-    import httpx
+    import httpx2
 
 
     @hookimpl
@@ -306,27 +316,18 @@ As an example, here's a very simple plugin which executes an HTTP response and r
             </form>""")
         vars = await request.post_vars()
         url = vars["url"]
-        return Response.text(httpx.get(url).text)
+        return Response.text(httpx2.get(url).text)
 
-Here's a test for that plugin that mocks the HTTPX outbound request:
+Here's a test for that plugin that mocks the HTTPX2 outbound request:
 
 .. code-block:: python
 
     from datasette.app import Datasette
-    import pytest
 
 
-    @pytest.fixture
-    def non_mocked_hosts():
-        # This ensures httpx-mock will not affect Datasette's own
-        # httpx calls made in the tests by datasette.client:
-        return ["localhost"]
-
-
-    async def test_outbound_http_call(httpx_mock):
-        httpx_mock.add_response(
-            url="https://www.example.com/",
-            text="Hello world",
+    async def test_outbound_http_call(httpx2_mock):
+        httpx2_mock.get("https://www.example.com/").respond(
+            text="Hello world"
         )
         datasette = Datasette([], memory=True)
         response = await datasette.client.post(
@@ -335,11 +336,13 @@ Here's a test for that plugin that mocks the HTTPX outbound request:
         )
         assert response.text == "Hello world"
 
-        outbound_request = httpx_mock.get_request()
+        outbound_request = httpx2_mock.calls.last.request
         assert (
             outbound_request.url == "https://www.example.com/"
         )
 
+If your plugin still makes its outbound calls using the original ``httpx`` library you can continue to mock those using `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__.
+
 .. _testing_plugins_register_in_test:
 
 Registering a plugin for the duration of a test

docs/writing_plugins.rst

@@ -203,6 +203,40 @@ Templates should be bundled for distribution using the same ``package_data`` mec
 
 You can also use wildcards here such as ``templates/*.html``. See `datasette-edit-schema <https://github.com/simonw/datasette-edit-schema>`__ for an example of this pattern.
 
+.. _writing_plugins_custom_templates_breadcrumbs:
+
+Adding breadcrumbs
+~~~~~~~~~~~~~~~~~~
+
+Plugin templates that extend ``base.html`` can use the ``crumbs.nav()`` macro to display breadcrumb links back to the Datasette homepage, and optionally to a database and a table. Override the ``crumbs`` block to specify which links to include:
+
+.. code-block:: html+jinja
+
+    {% extends "base.html" %}
+
+    {% block title %}Manage {{ table }}{% endblock %}
+
+    {% block crumbs %}
+    <!-- For home / database / table -->
+    {{ crumbs.nav(request=request, database=database, table=table) }}
+    <!-- For home / database -->
+    {{ crumbs.nav(request=request, database=database) }}
+    {% endblock %}
+
+    {% block content %}
+    <h1>Manage {{ table }}</h1>
+    {% endblock %}
+
+The macro accepts these arguments:
+
+* ``request``: the current request, used to check the actor's permissions.
+* ``database``: an optional database name, as a string
+* ``table``: an optional table name, as a string. If you pass ``table``, you must also pass ``database``.
+
+For a database-level plugin page, use ``{{ crumbs.nav(request=request, database=database) }}``. For a page with just a homepage link, use ``{{ crumbs.nav(request=request) }}``, which is also the default provided by ``base.html`` if you do not override the block.
+
+The table-level example renders links in the form ``home / database / table``. Each link is only included if the current actor has permission to view that resource.
+
 .. _writing_plugins_configuration:
 
 Writing plugins that accept configuration

pyproject.toml

@@ -28,9 +28,9 @@ dependencies = [
     "click-default-group>=1.2.3",
     "Jinja2>=2.10.3",
     "hupper>=1.9",
-    "httpx>=0.20,<1.0",
+    "httpx2>=2.0",
     "pluggy>=1.0",
-    "uvicorn>=0.11",
+    "uvicorn>=0.29",
     "aiofiles>=0.4",
     "PyYAML>=5.3",
     "mergedeep>=1.1.1",

pytest.ini

@@ -1,4 +1,5 @@
 [pytest]
+addopts = --ignore=ignored
 filterwarnings=
     # https://github.com/pallets/jinja/issues/927
     ignore:Using or importing the ABCs::jinja2

tests/conftest.py

@@ -2,13 +2,14 @@ import importlib.metadata
 import os
 import pathlib
 import re
+import socket
 import subprocess
 import sys
 import tempfile
 import time
 from dataclasses import dataclass
 
-import httpx
+import httpx2
 import pytest
 import pytest_asyncio
 
@@ -32,17 +33,31 @@ UNDOCUMENTED_PERMISSIONS = {
 }
 
 
-def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs):
+def wait_until_responds(url, timeout=5.0, client=httpx2, process=None, **kwargs):
     start = time.time()
     while time.time() - start < timeout:
+        # If the server died there is no point waiting out the timeout - fail
+        # now, with its output, instead of after `timeout` seconds of silence
+        if process is not None and process.poll() is not None:
+            raise AssertionError(
+                "Server exited early with returncode {}\n{}".format(
+                    process.returncode, process.stdout.read().decode("utf-8")
+                )
+            )
         try:
             client.get(url, **kwargs)
             return
-        except httpx.ConnectError:
+        except httpx2.TransportError:
             time.sleep(0.1)
     raise AssertionError(f"Timed out waiting for {url} to respond")
 
 
+def find_free_port():
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
 @pytest.fixture
 def bare_ds():
     """
@@ -91,7 +106,10 @@ async def ds_client():
 
     await db.execute_write_fn(prepare)
     await ds.invoke_startup()
-    return ds.client
+    try:
+        yield ds.client
+    finally:
+        ds.close()
 
 
 def pytest_report_header(config):
@@ -251,12 +269,24 @@ def ds_localhost_http_server():
         # Avoid FileNotFoundError: [Errno 2] No such file or directory:
         cwd=tempfile.gettempdir(),
     )
-    wait_until_responds("http://localhost:8041/")
-    # Check it started successfully
-    assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8")
+    try:
+        wait_until_responds("http://localhost:8041/", process=ds_proc)
         yield ds_proc
-    # Shut it down at the end of the pytest session
-    ds_proc.terminate()
+    finally:
+        stop_process(ds_proc)
+
+
+def stop_process(proc):
+    try:
+        if proc.poll() is None:
+            proc.terminate()
+        try:
+            proc.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            proc.kill()
+            proc.wait()
+    finally:
+        proc.stdout.close()
 
 
 @pytest.fixture(scope="session")
@@ -277,11 +307,27 @@ def ds_unix_domain_socket_server(tmp_path_factory):
         cwd=tempfile.gettempdir(),
     )
     # Poll until available
-    transport = httpx.HTTPTransport(uds=uds)
-    client = httpx.Client(transport=transport)
+    transport = httpx2.HTTPTransport(uds=uds)
+    client = httpx2.Client(transport=transport)
     try:
+        # Probe with a socket we own: the HTTP transport can leak a socket
+        # when connect() fails before the UDS server has started listening.
+        start = time.monotonic()
+        while True:
+            try:
+                with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as probe:
+                    probe.settimeout(0.1)
+                    probe.connect(uds)
+                break
+            except OSError:
+                if ds_proc.poll() is not None or time.monotonic() - start > 30:
+                    raise
+                time.sleep(0.1)
         wait_until_responds(
-            "http://localhost/_memory.json", timeout=30.0, client=client
+            "http://localhost/_memory.json",
+            timeout=30.0,
+            client=client,
+            process=ds_proc,
         )
         # Check it started successfully
         assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8")
@@ -289,18 +335,72 @@ def ds_unix_domain_socket_server(tmp_path_factory):
     finally:
         client.close()
         # Shut it down at the end of the pytest session
-        ds_proc.terminate()
-        try:
-            ds_proc.wait(timeout=5)
-        except subprocess.TimeoutExpired:
-            ds_proc.kill()
-            ds_proc.wait()
+        stop_process(ds_proc)
         try:
             os.unlink(uds)
         except FileNotFoundError:
             pass
 
 
+@pytest.fixture
+def serve_with_plugins(tmp_path):
+    """Factory fixture for starting ``datasette serve`` in a subprocess with
+    plugins written to a temporary ``--plugins-dir``.
+
+    For tests that need the real serve path: event-loop wiring, exit codes,
+    signals. The usual in-process ``pm.register`` plugin pattern can't reach
+    a subprocess, so plugin source is written out as importable files instead.
+
+    Unlike ``ds_localhost_http_server`` this is function-scoped and takes a
+    fresh port each time, because each test needs its own plugins. Call it as::
+
+        proc, port = serve_with_plugins({"my_plugin": PLUGIN_SOURCE})
+
+    ``plugins`` maps module name to Python source. Pass
+    ``wait_for_startup=False`` when the server is expected to fail during
+    startup rather than begin serving. Extra CLI arguments are passed through.
+    Every process started is terminated when the test ends.
+    """
+    processes = []
+
+    def start(plugins, *extra_args, wait_for_startup=True):
+        plugins_dir = tmp_path / "plugins"
+        plugins_dir.mkdir(exist_ok=True)
+        for module_name, source in plugins.items():
+            (plugins_dir / f"{module_name}.py").write_text(source, "utf-8")
+        port = find_free_port()
+        proc = subprocess.Popen(
+            [
+                sys.executable,
+                "-m",
+                "datasette",
+                "--memory",
+                "--plugins-dir",
+                str(plugins_dir),
+                "-h",
+                "127.0.0.1",
+                "-p",
+                str(port),
+                *extra_args,
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            # Avoid FileNotFoundError: [Errno 2] No such file or directory:
+            cwd=tempfile.gettempdir(),
+        )
+        processes.append(proc)
+        if wait_for_startup:
+            wait_until_responds(
+                f"http://127.0.0.1:{port}/-/versions.json", process=proc
+            )
+        return proc, port
+
+    yield start
+
+    for proc in processes:
+        stop_process(proc)
+
+
 # Import fixtures from fixtures.py to make them available
 from .fixtures import (  # noqa: F401
     TEMP_PLUGIN_SECRET_FILE,

tests/fixtures.py

@@ -169,12 +169,10 @@ def make_app_client(
             template_dir=template_dir,
             crossdb=crossdb,
         )
+        try:
             yield TestClient(ds)
-        # Close as many database connections as possible
-        # to try and avoid too many open files error
-        for db in ds.databases.values():
-            if not db.is_memory:
-                db.close()
+        finally:
+            ds.close()
 
 
 @pytest.fixture(scope="session")
@@ -186,9 +184,10 @@ def app_client():
 @pytest.fixture(scope="session")
 def app_client_no_files():
     ds = Datasette([])
+    try:
         yield TestClient(ds)
-    for db in ds.databases.values():
-        db.close()
+    finally:
+        ds.close()
 
 
 @pytest.fixture(scope="session")

tests/test_api.py

@@ -5,6 +5,7 @@ import pytest
 
 from datasette.app import Datasette
 from datasette.plugins import DEFAULT_PLUGINS
+from datasette.resources import DatabaseResource, TableResource
 from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode
 from datasette.utils.sqlite import sqlite_version
 from datasette.version import __version__
@@ -101,14 +102,11 @@ async def test_database_page(ds_client):
         "tags",
     }
 
-    # Expected hidden tables
+    # The external-content index is visible, but its shadow tables need a
+    # second dependency hop and are excluded by the one-hop permission policy.
     expected_hidden_tables = {
         "no_primary_key",
         "searchable_fts",
-        "searchable_fts_config",
-        "searchable_fts_data",
-        "searchable_fts_docsize",
-        "searchable_fts_idx",
     }
 
     # Verify all expected tables exist
@@ -458,6 +456,67 @@ async def test_row_foreign_key_tables(ds_client):
     ]
 
 
+@pytest.mark.asyncio
+async def test_row_foreign_key_tables_omit_denied_tables(request):
+    actor = {"id": "reader"}
+    ds = Datasette(
+        memory=True,
+        default_deny=True,
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "parents": {"permissions": {"view-table": True}},
+                        "private_children": {"permissions": {"view-table": False}},
+                    }
+                }
+            }
+        },
+    )
+    request.addfinalizer(ds.close)
+    db = ds.add_memory_database("fk_count_leak", name="data")
+    await db.execute_write("create table parents (id integer primary key, name text)")
+    await db.execute_write("""
+        create table private_children (
+            id integer primary key,
+            parent_id integer references parents(id)
+        )
+    """)
+    await db.execute_write("insert into parents values (1, 'Public parent')")
+    await db.execute_write("""
+        insert into private_children (id, parent_id) values
+            (1, 1),
+            (2, 1),
+            (3, 1)
+    """)
+    await ds.invoke_startup()
+
+    parent = TableResource(database="data", table="parents")
+    private_children = TableResource(database="data", table="private_children")
+    assert await ds.allowed(action="view-table", resource=parent, actor=actor)
+    assert not await ds.allowed(
+        action="view-table", resource=private_children, actor=actor
+    )
+    assert not await ds.allowed(
+        action="execute-sql",
+        resource=DatabaseResource(database="data"),
+        actor=actor,
+    )
+
+    direct_child = await ds.client.get("/data/private_children.json", actor=actor)
+    assert direct_child.status_code == 403
+    parent_response = await ds.client.get(
+        "/data/parents/1.json?_extra=foreign_key_tables", actor=actor
+    )
+    assert parent_response.status_code == 200
+
+    foreign_key_tables = parent_response.json().get("foreign_key_tables", [])
+    assert foreign_key_tables == [], (
+        "denied child table name, foreign-key column, and row count disclosed: "
+        f"{foreign_key_tables}"
+    )
+
+
 @pytest.mark.asyncio
 async def test_row_extras(ds_client):
     response = await ds_client.get(
@@ -894,10 +953,7 @@ async def test_hidden_sqlite_stat1_table():
     await db.execute_write("analyze")
     data = (await ds.client.get("/db.json?_show_hidden=1")).json()
     tables = [(t["name"], t["hidden"]) for t in data["tables"]]
-    assert tables in (
-        [("normal", False), ("sqlite_stat1", True)],
-        [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)],
-    )
+    assert tables == [("normal", False)]
 
 
 @pytest.mark.asyncio

tests/test_api_write.py

@@ -56,6 +56,105 @@ def _headers(token):
     }
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("operation", ["read", "read_row", "rename"])
+async def test_trailing_lf_table_permissions(tmp_path, operation):
+    # SQLite treats "secret" and "secret\n" as different table names. Permission
+    # checks and SQL execution must agree on which table a request targets.
+    db_path = tmp_path / "data.db"
+    conn = sqlite3.connect(str(db_path))
+    conn.executescript(
+        "create table secret (id integer primary key, value text);"
+        "insert into secret values (1, 'private');"
+    )
+    conn.close()
+    # Allow builder to create and use tables generally, but explicitly deny
+    # access to the existing secret table below. Disable arbitrary SQL access.
+    grants = {
+        action: {"id": "builder"}
+        for action in (
+            "view-database",
+            "create-table",
+            "view-table",
+            "insert-row",
+            "alter-table",
+        )
+    }
+    ds = Datasette(
+        [str(db_path)],
+        default_deny=True,
+        settings={"default_allow_sql": False},
+        config={
+            "permissions": {"view-instance": {"id": "builder"}},
+            "databases": {
+                "data": {
+                    "permissions": grants,
+                    "tables": {
+                        "secret": {
+                            "permissions": {
+                                "view-table": False,
+                                "insert-row": False,
+                                "alter-table": False,
+                            }
+                        }
+                    },
+                }
+            },
+        },
+    )
+    headers = _headers(write_token(ds, actor_id="builder"))
+    try:
+        # Establish that the protected table is inaccessible before creating
+        # a second table whose name differs only by a trailing line feed.
+        response = await ds.client.get("/data/secret.json", headers=headers)
+        assert response.status_code == 403
+        response = await ds.client.get(
+            "/data/-/query.json?sql=select+*+from+secret", headers=headers
+        )
+        assert response.status_code == 403
+        # Distinct values let us detect if an operation targets secret
+        # instead of the newly created secret\n table.
+        response = await ds.client.post(
+            "/data/-/create",
+            json={"table": "secret\n", "row": {"id": 1, "value": "decoy"}, "pk": "id"},
+            headers=headers,
+        )
+        assert response.status_code == 201, response.text
+        # ~0A is Datasette's URL encoding for the line feed in the table name.
+        if operation in ("read", "read_row"):
+            # Both table and row endpoints must return only the permitted row.
+            path = "/1.json" if operation == "read_row" else ".json"
+            response = await ds.client.get(
+                "/data/secret~0A" + path + "?_shape=array", headers=headers
+            )
+            assert response.status_code == 200, response.text
+            assert response.json() == [{"id": 1, "value": "decoy"}]
+        else:
+            # Renaming must move the permitted table, preserving its contents
+            # and removing its old name from the database.
+            response = await ds.client.post(
+                "/data/secret~0A/-/alter",
+                json={"operations": [{"op": "rename_table", "args": {"to": "moved"}}]},
+                headers=headers,
+            )
+            assert response.status_code == 200, response.text
+            db = ds.get_database("data")
+            assert (
+                await db.execute('select value from "moved"')
+            ).single_value() == "decoy"
+            assert "secret\n" not in await db.table_names()
+        # Verify that the protected table and its data are unchanged, and that
+        # the API still denies access to it.
+        db = ds.get_database("data")
+        assert (
+            await db.execute('select value from "secret"')
+        ).single_value() == "private"
+        response = await ds.client.get("/data/secret.json", headers=headers)
+        assert response.status_code == 403
+    finally:
+        ds.close()
+
+
 def _insert_and_fetch_created(conn, table, insert_sql):
     cursor = conn.execute(insert_sql)
     return conn.execute(
@@ -68,6 +167,82 @@ BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"}
 BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}'
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_fallback", (False, True))
+@pytest.mark.parametrize(
+    "operation", ("insert", "upsert", "update", "delete", "create", "create_uppercase")
+)
+@pytest.mark.parametrize(
+    "module,definition,values,shadow_suffix",
+    (
+        ("fts5", "body", "'original'", "_content"),
+        ("fts4", "body", "'original'", "_content"),
+        ("rtree", "id, minx, maxx", "1, 0, 1", "_rowid"),
+    ),
+)
+@pytest.mark.parametrize("shadow", (False, True))
+async def test_structured_writes_require_ordinary_tables(
+    ds_write,
+    monkeypatch,
+    use_fallback,
+    operation,
+    module,
+    definition,
+    values,
+    shadow_suffix,
+    shadow,
+):
+    if use_fallback:
+        monkeypatch.setattr("datasette.utils.sqlite.supports_table_list", lambda: False)
+    db = ds_write.get_database("data")
+    await db.execute_write(f"create virtual table indexed using {module}({definition})")
+    await db.execute_write(f"insert into indexed values ({values})")
+    table = "indexed" + (shadow_suffix if shadow else "")
+    row = (await db.execute(f"select rowid, * from {escape_sqlite(table)}")).dicts()[0]
+    pks = await db.primary_keys(table)
+    pk_value = row[pks[0] if pks else "rowid"]
+    before = await db.execute_fn(lambda conn: list(conn.iterdump()))
+
+    if operation in ("create", "create_uppercase"):
+        path = "/data/-/create"
+        body = {
+            "table": table.upper() if operation == "create_uppercase" else table,
+            "rows": [row],
+        }
+    elif operation in ("update", "delete"):
+        path = f"/data/{table}/{pk_value}/-/{operation}"
+        body = {"update": row} if operation == "update" else {}
+    else:
+        path = f"/data/{table}/-/{operation}"
+        body = {"rows": [row]}
+    response = await ds_write.client.post(
+        path, json=body, headers=_headers(write_token(ds_write))
+    )
+    assert response.status_code == 400, response.text
+    assert response.json()["errors"] == ["Structured writes require an ordinary table"]
+    assert await db.execute_fn(lambda conn: list(conn.iterdump())) == before
+
+
+@pytest.mark.asyncio
+async def test_structured_writes_to_content_table_maintain_fts(ds_write):
+    db = ds_write.get_database("data")
+    await db.execute_write_fn(
+        lambda conn: sqlite_utils.Database(conn)["docs"].enable_fts(
+            ["title"], create_triggers=True
+        )
+    )
+    response = await ds_write.client.post(
+        "/data/docs/-/insert",
+        json={"row": {"id": 1, "title": "ordinary content"}},
+        headers=_headers(write_token(ds_write)),
+    )
+    assert response.status_code == 201, response.text
+    matches = await db.execute(
+        "select rowid from docs_fts where docs_fts match ?", ["ordinary"]
+    )
+    assert [row[0] for row in matches.rows] == [1]
+
+
 @pytest.mark.asyncio
 async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write):
     token = write_token(ds_write)
@@ -1296,7 +1471,7 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w
 
 @pytest.mark.asyncio
 async def test_foreign_key_suggestions(ds_write):
-    token = write_token(ds_write, permissions=["at"])
+    token = write_token(ds_write, permissions=["alter-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
     await db.execute_write("insert into owners (id) values (1), (2), (3)")
@@ -1362,7 +1537,7 @@ async def test_foreign_key_suggestions_permission_denied(ds_write):
 
 @pytest.mark.asyncio
 async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch):
-    token = write_token(ds_write, permissions=["at"])
+    token = write_token(ds_write, permissions=["alter-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
 
@@ -1393,7 +1568,7 @@ async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch):
 
 @pytest.mark.asyncio
 async def test_foreign_key_targets(ds_write):
-    token = write_token(ds_write, permissions=["ct"])
+    token = write_token(ds_write, permissions=["create-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
     await db.execute_write("create table categories (slug varchar(30) primary key)")
@@ -2745,3 +2920,119 @@ async def test_create_using_alter_against_existing_table(
         insert_rows_event = ds_write._tracked_events[1]
         assert insert_rows_event.name == "insert-rows"
         assert insert_rows_event.num_rows == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("denied_action", "request_body"),
+    (
+        (
+            "insert-row",
+            {
+                "table": "salaries",
+                "rows": [{"id": 9, "note": "INJ-VIA-CREATE"}],
+            },
+        ),
+        (
+            "update-row",
+            {
+                "table": "salaries",
+                "rows": [{"id": 1, "note": "REPLACED"}],
+                "pk": "id",
+                "replace": True,
+            },
+        ),
+        (
+            "alter-table",
+            {
+                "table": "salaries",
+                "rows": [{"id": 9, "note": "INSERTED", "extra": "NEW"}],
+                "alter": True,
+            },
+        ),
+    ),
+)
+async def test_create_table_existing_table_respects_table_level_denial(
+    denied_action, request_body
+):
+    # GHSA-53fc-rhfg-h7qp issue 2: POST /db/-/create against an existing table
+    # inserts rows into it, so insert-row (and update-row / alter-table) must be
+    # checked against the TableResource, not just the DatabaseResource.
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                # id=editor user has each permission at the database level, but
+                # the selected action is explicitly denied on the salaries table
+                "data": {
+                    "permissions": {
+                        "create-table": {"id": "editor"},
+                        "insert-row": {"id": "editor"},
+                        "update-row": {"id": "editor"},
+                        "alter-table": {"id": "editor"},
+                    },
+                    "tables": {
+                        "salaries": {"permissions": {denied_action: False}},
+                    },
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database(
+        f"create_table_existing_table_denied_{denied_action}", name="data"
+    )
+    await db.execute_write("create table salaries (id integer primary key, note text)")
+    await db.execute_write("insert into salaries values (1, 'TOPSECRET-A')")
+    await ds.invoke_startup()
+
+    if denied_action == "insert-row":
+        # Sanity: direct insert into salaries is denied for this actor
+        direct = await ds.client.post(
+            "/data/salaries/-/insert",
+            actor={"id": "editor"},
+            json={"row": {"id": 9, "note": "INJ-DIRECT"}},
+        )
+        assert direct.status_code == 403
+
+    response = await ds.client.post(
+        "/data/-/create",
+        actor={"id": "editor"},
+        json=request_body,
+    )
+    assert response.status_code == 403, response.json()
+    assert response.json()["errors"] == [f"Permission denied: need {denied_action}"]
+    rows = (await db.execute("select id, note from salaries order by id")).rows
+    assert [tuple(r) for r in rows] == [(1, "TOPSECRET-A")]
+    assert await db.table_columns("salaries") == ["id", "note"]
+
+
+@pytest.mark.asyncio
+async def test_create_table_respects_predeclared_table_level_denial():
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                "data": {
+                    "permissions": {
+                        "create-table": {"id": "editor"},
+                        "insert-row": {"id": "editor"},
+                    },
+                    "tables": {
+                        "planned_table": {"permissions": {"insert-row": False}},
+                    },
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database("create_table_predeclared_denial", name="data")
+    await ds.invoke_startup()
+
+    response = await ds.client.post(
+        "/data/-/create",
+        actor={"id": "editor"},
+        json={"table": "planned_table", "rows": [{"id": 1}]},
+    )
+
+    assert response.status_code == 403, response.json()
+    assert response.json()["errors"] == ["Permission denied: need insert-row"]
+    assert not await db.table_exists("planned_table")

tests/test_auth.py

@@ -1,4 +1,5 @@
 import time
+from unittest.mock import AsyncMock
 
 import pytest
 from bs4 import BeautifulSoup as Soup
@@ -237,6 +238,35 @@ def test_auth_create_token(
         assert response3.json["actor"]["id"] == "test"
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["GET", "POST"])
+@pytest.mark.parametrize(
+    "restrictions",
+    [
+        {},
+        {"a": ["vi"]},
+        {"d": {"db": ["vd"]}},
+        {"r": {"db": {"t1": ["vt"]}}},
+    ],
+    ids=["empty", "instance", "database", "table"],
+)
+async def test_auth_create_token_not_allowed_for_restricted_actors(
+    bare_ds, monkeypatch, method, restrictions
+):
+    create_token = AsyncMock()
+    monkeypatch.setattr(bare_ds, "create_token", create_token)
+
+    response = await bare_ds.client.request(
+        method,
+        "/-/create-token",
+        actor={"id": "test", "_r": restrictions},
+    )
+
+    assert response.status_code == 403
+    assert "Restricted actors cannot create API tokens" in response.text
+    create_token.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_auth_create_token_not_allowed_for_tokens(ds_client):
     ds_tok = ds_client.ds.sign(
@@ -524,3 +554,25 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client):
         )
         is not True
     ), "Root without root_enabled should not automatically get set-column-type"
+
+
+@pytest.mark.parametrize("expire_after", (1, 300, 3600, 30 * 24 * 60 * 60))
+def test_set_actor_cookie_honours_expire_after(expire_after):
+    # GHSA-53fc-rhfg-h7qp issue 4: expire_after is documented as a number of
+    # seconds, but every value was being replaced with 24 hours.
+    from datasette.app import Datasette
+    from datasette.utils.asgi import Response
+
+    ds = Datasette(memory=True)
+    response = Response.text("")
+    before = int(time.time())
+    ds.set_actor_cookie(response, {"id": "test"}, expire_after=expire_after)
+    after = int(time.time())
+
+    (header,) = response._set_cookie_headers
+    assert header.startswith("ds_actor=")
+    value = header[len("ds_actor=") :].split(";", 1)[0]
+    data = ds.unsign(value, "actor")
+    assert data["a"] == {"id": "test"}
+    expires_at = baseconv.base62.decode(data["e"])
+    assert before + expire_after <= expires_at <= after + expire_after

tests/test_background_tasks.py

@@ -0,0 +1,381 @@
+"""
+Tests for datasette.add_background_task() / start_background_tasks() and the
+BackgroundTask / BackgroundTaskSupervisor machinery in
+datasette/background_tasks.py.
+"""
+
+import asyncio
+import contextlib
+import logging
+
+import httpx2
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.plugins import pm
+
+
+async def _drive_lifespan_startup(app):
+    """Send a single lifespan.startup message into app's ASGI lifespan loop
+    and return the list of messages sent back, without ever sending
+    lifespan.shutdown. Copied from tests/test_lifespan.py's helper of the
+    same name - mirrors what a real server does: after startup completes
+    it parks waiting for the next event, and we cancel that wait once
+    we've observed the startup response.
+    """
+    messages_sent = []
+    startup_responded = asyncio.Event()
+    delivered = False
+
+    async def receive():
+        nonlocal delivered
+        if not delivered:
+            delivered = True
+            return {"type": "lifespan.startup"}
+        await asyncio.Event().wait()
+
+    async def send(message):
+        messages_sent.append(message)
+        startup_responded.set()
+
+    task = asyncio.create_task(app({"type": "lifespan"}, receive, send))
+    try:
+        await asyncio.wait_for(startup_responded.wait(), timeout=5)
+    finally:
+        task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await task
+    return messages_sent
+
+
+@pytest.mark.asyncio
+async def test_tasks_registered_in_startup_hook_run_after_lifespan_startup():
+    # Two tasks registered by one plugin's startup hook - order preserved,
+    # both running after lifespan startup completes, and no HTTP request
+    # of any kind is issued anywhere in this test.
+    events = []
+
+    async def task_one(datasette):
+        events.append("task_one")
+        # Wait indefinitely to simulate long-lived background work, keeping
+        # the task "running" for the assertions below until cleanup cancels it.
+        await asyncio.Event().wait()
+
+    async def task_two(datasette):
+        events.append("task_two")
+        await asyncio.Event().wait()
+
+    class TwoTaskPlugin:
+        __name__ = "TwoTaskPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(task_one, name="task-one")
+                datasette.add_background_task(task_two, name="task-two")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(TwoTaskPlugin(), name="two_task_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan_startup(app)
+        assert {"type": "lifespan.startup.complete"} in messages
+
+        handles = ds._background_tasks.tasks()
+        assert [h.name for h in handles] == ["task-one", "task-two"]
+
+        # Let both tasks run their first line of code.
+        await asyncio.sleep(0)
+        assert handles[0].state == "running"
+        assert handles[1].state == "running"
+        assert events == ["task_one", "task_two"]
+    finally:
+        pm.unregister(name="two_task_plugin")
+        await ds._background_tasks.cancel_all(grace=1.0)
+
+
+@pytest.mark.asyncio
+async def test_launch_waits_for_every_startup_hook_before_running_any_task():
+    # PluginA registers a task from its startup hook; PluginB does the
+    # same from ITS startup hook, which runs after PluginA's (forced with
+    # tryfirst=True on A). Even though A's registration happens first,
+    # A's task body must not actually execute until every startup hook -
+    # including B's - has finished, since launch only happens after
+    # invoke_startup() completes. This is the ordering guarantee that
+    # dissolves datasette-cron's tryfirst=True launch hack.
+    hook_call_order = []
+    seen_names_when_a_ran = {}
+
+    async def task_a(datasette):
+        seen_names_when_a_ran["names"] = [
+            h.name for h in datasette._background_tasks.tasks()
+        ]
+
+    async def task_b(datasette):
+        pass
+
+    class PluginA:
+        __name__ = "PluginA"
+
+        @hookimpl(tryfirst=True)
+        def startup(self, datasette):
+            async def inner():
+                hook_call_order.append("A")
+                datasette.add_background_task(task_a, name="task-a")
+
+            return inner
+
+    class PluginB:
+        __name__ = "PluginB"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                hook_call_order.append("B")
+                datasette.add_background_task(task_b, name="task-b")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(PluginA(), name="plugin_a")
+    pm.register(PluginB(), name="plugin_b")
+    try:
+        await ds.start_background_tasks()
+        # Confirm A's startup hook really did run (and register task-a)
+        # strictly before B's startup hook ran.
+        assert hook_call_order == ["A", "B"]
+
+        handles = ds._background_tasks.tasks()
+        await asyncio.wait_for(asyncio.gather(*[h.task for h in handles]), timeout=5)
+        # Yet by the time task-a's own body executed (after launch, which
+        # only happens once every startup hook - including B's - has
+        # finished), task-b was already registered.
+        assert "task-b" in seen_names_when_a_ran["names"]
+    finally:
+        pm.unregister(name="plugin_a")
+        pm.unregister(name="plugin_b")
+
+
+@pytest.mark.asyncio
+async def test_concurrent_first_requests_launch_background_tasks_exactly_once():
+    launch_count = {"n": 0}
+
+    async def counting_task(datasette):
+        launch_count["n"] += 1
+
+    class CountingTaskPlugin:
+        __name__ = "CountingTaskPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(counting_task, name="counting-task")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(CountingTaskPlugin(), name="counting_task_plugin")
+    try:
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            responses = await asyncio.gather(
+                *[client.get("/-/versions.json") for _ in range(10)]
+            )
+        assert all(response.status_code == 200 for response in responses)
+
+        handles = ds._background_tasks.tasks()
+        assert len(handles) == 1
+        await asyncio.wait_for(handles[0].task, timeout=5)
+        assert launch_count["n"] == 1
+    finally:
+        pm.unregister(name="counting_task_plugin")
+
+
+@pytest.mark.asyncio
+async def test_post_launch_registration_starts_immediately_and_cancel_works():
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()  # nothing registered yet, but launched
+
+    started = asyncio.Event()
+
+    async def long_running(datasette):
+        started.set()
+        await asyncio.Event().wait()
+
+    handle = ds.add_background_task(long_running, name="dynamic-task")
+    # Registered after launch: starts immediately rather than sitting in
+    # "registered" limbo.
+    assert handle.state == "running"
+    assert handle.task is not None
+
+    await asyncio.wait_for(started.wait(), timeout=5)
+    assert handle.state == "running"
+
+    handle.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await handle.task
+    await asyncio.sleep(0)
+    assert handle.state == "cancelled"
+
+
+@pytest.mark.asyncio
+async def test_pre_launch_registration_starts_as_registered():
+    ds = Datasette(memory=True)
+
+    async def task(datasette):
+        pass
+
+    handle = ds.add_background_task(task, name="buffered-task")
+    assert handle.state == "registered"
+    assert handle.task is None
+
+    handle.cancel()  # not yet launched: deregisters instead of cancelling
+    assert handle not in ds._background_tasks.tasks()
+
+
+@pytest.mark.asyncio
+async def test_crashing_task_logs_traceback_and_state_is_crashed(caplog):
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()
+
+    survivor_ran = asyncio.Event()
+
+    async def crashing_task(datasette):
+        raise RuntimeError("kaboom")
+
+    async def survivor(datasette):
+        survivor_ran.set()
+
+    with caplog.at_level(logging.ERROR, logger="datasette.background_tasks"):
+        crash_handle = ds.add_background_task(crashing_task, name="crashing_task")
+        survivor_handle = ds.add_background_task(survivor, name="survivor")
+        await asyncio.wait_for(
+            asyncio.gather(
+                crash_handle.task, survivor_handle.task, return_exceptions=True
+            ),
+            timeout=5,
+        )
+
+    assert crash_handle.state == "crashed"
+    assert isinstance(crash_handle.exception, RuntimeError)
+    assert str(crash_handle.exception) == "kaboom"
+
+    # The crash must not affect any other task.
+    assert survivor_ran.is_set()
+    assert survivor_handle.state == "completed"
+
+    assert "crashing_task" in caplog.text
+    assert "kaboom" in caplog.text
+    assert "Traceback" in caplog.text
+    assert "RuntimeError" in caplog.text
+
+
+def test_name_collisions_get_suffixed_and_explicit_names_are_respected():
+    ds = Datasette(memory=True)
+
+    async def noop(datasette):
+        pass
+
+    async def another_noop(datasette):
+        pass
+
+    h1 = ds.add_background_task(noop, name="dup")
+    h2 = ds.add_background_task(another_noop, name="dup")
+    h3 = ds.add_background_task(noop, name="dup")
+    assert [h1.name, h2.name, h3.name] == ["dup", "dup-2", "dup-3"]
+
+    h_explicit = ds.add_background_task(noop, name="explicit-name")
+    assert h_explicit.name == "explicit-name"
+
+    h_default = ds.add_background_task(noop)
+    assert h_default.name == noop.__qualname__
+
+
+@pytest.mark.asyncio
+async def test_start_background_tasks_on_bare_datasette():
+    # The headless-CLI path (datasette-rss's `fetch --due` shape): no
+    # server, no lifespan, no first HTTP request - just an explicit call.
+    ran = asyncio.Event()
+
+    async def task(datasette):
+        ran.set()
+
+    ds = Datasette([])
+    assert ds._startup_invoked is False
+
+    handle = ds.add_background_task(task, name="headless-task")
+    assert handle.state == "registered"
+
+    await ds.start_background_tasks()
+
+    assert ds._startup_invoked is True
+    await asyncio.wait_for(ran.wait(), timeout=5)
+    await asyncio.wait_for(handle.task, timeout=5)
+    # handle.task being done only guarantees the coroutine has returned,
+    # not that our done-callback (which updates handle.state) has run yet -
+    # asyncio schedules done-callbacks via call_soon, and awaiting an
+    # already-done future/task returns immediately without giving the loop
+    # a chance to drain its ready queue. Yield once to let it run.
+    await asyncio.sleep(0)
+    assert handle.state == "completed"
+
+
+@pytest.mark.asyncio
+async def test_cancel_all_cancels_running_tasks_and_leaves_completed_alone():
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()
+
+    async def forever(datasette):
+        await asyncio.Event().wait()
+
+    async def quick(datasette):
+        return "done"
+
+    forever_handle = ds.add_background_task(forever, name="forever")
+    quick_handle = ds.add_background_task(quick, name="quick")
+    await asyncio.wait_for(quick_handle.task, timeout=5)
+    assert quick_handle.state == "completed"
+
+    await ds._background_tasks.cancel_all(grace=1.0)
+
+    assert forever_handle.state == "cancelled"
+    assert quick_handle.state == "completed"
+
+
+@pytest.mark.asyncio
+async def test_cancel_all_logs_stragglers_that_outlive_the_grace_period(caplog):
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()
+
+    async def stubborn(datasette):
+        with contextlib.suppress(asyncio.CancelledError):
+            await asyncio.sleep(10)
+        # Swallowing CancelledError above and returning normally simulates
+        # a task that ignores cancellation for longer than the grace period.
+        await asyncio.sleep(10)
+
+    handle = ds.add_background_task(stubborn, name="stubborn-task")
+    # Let the task actually start running and reach its first sleep (inside
+    # the CancelledError-suppressing block) before cancelling it - a task
+    # cancelled before it has ever run its first step never enters that
+    # block at all (the throw happens before the coroutine body starts),
+    # so it would finish cancelling immediately instead of behaving like a
+    # straggler.
+    await asyncio.sleep(0)
+
+    with caplog.at_level(logging.WARNING, logger="datasette.background_tasks"):
+        await ds._background_tasks.cancel_all(grace=0.1)
+
+    assert "stubborn-task" in caplog.text
+
+    # Clean up: actually cancel it now that the test has made its
+    # assertion, so it doesn't leak past the end of the test.
+    handle.task.cancel()
+    with contextlib.suppress(asyncio.CancelledError):
+        await handle.task

tests/test_cli_serve_get.py

@@ -52,6 +52,56 @@ def test_serve_with_get(tmp_path_factory):
     pm.unregister(to_unregister)
 
 
+def test_serve_with_get_does_not_launch_background_tasks(tmp_path_factory):
+    # --get must never launch background tasks, even though its TestClient
+    # request
+    # flows through the full ASGI stack (including the AsgiRunOnFirstRequest
+    # fallback that would otherwise launch them). The plugin's startup hook
+    # itself still runs (registration happens) - only the launch is
+    # suppressed, so the sentinel file the background task would write must
+    # never appear.
+    plugins_dir = tmp_path_factory.mktemp("plugins_for_get_background_tasks")
+    sentinel = plugins_dir / "sentinel.txt"
+    (plugins_dir / "bg_task_for_get.py").write_text(
+        textwrap.dedent(
+            f"""
+        from datasette import hookimpl
+
+        @hookimpl
+        def startup(datasette):
+            async def inner():
+                async def task(datasette):
+                    with open("{sentinel!s}", "w") as fp:
+                        fp.write("ran")
+
+                datasette.add_background_task(task, name="get-sentinel-task")
+
+            return inner
+    """,
+        ),
+        "utf-8",
+    )
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        [
+            "serve",
+            "--memory",
+            "--plugins-dir",
+            str(plugins_dir),
+            "--get",
+            "/_memory/-/query.json?sql=select+1",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert not sentinel.exists()
+
+    to_unregister = next(
+        p for p in pm.get_plugins() if p.__name__ == "bg_task_for_get.py"
+    )
+    pm.unregister(to_unregister)
+
+
 def test_serve_with_get_headers():
     runner = CliRunner()
     result = runner.invoke(

tests/test_cli_serve_server.py

@@ -1,12 +1,15 @@
+import signal
 import socket
+import subprocess
+import time
 
-import httpx
+import httpx2
 import pytest
 
 
 @pytest.mark.serial
 def test_serve_localhost_http(ds_localhost_http_server):
-    response = httpx.get("http://localhost:8041/_memory.json")
+    response = httpx2.get("http://localhost:8041/_memory.json")
     assert {
         "database": "_memory",
         "path": "/_memory",
@@ -20,11 +23,205 @@ def test_serve_localhost_http(ds_localhost_http_server):
 )
 def test_serve_unix_domain_socket(ds_unix_domain_socket_server):
     _, uds = ds_unix_domain_socket_server
-    transport = httpx.HTTPTransport(uds=uds)
-    client = httpx.Client(transport=transport)
+    transport = httpx2.HTTPTransport(uds=uds)
+    with httpx2.Client(transport=transport) as client:
         response = client.get("http://localhost/_memory.json")
     assert {
         "database": "_memory",
         "path": "/_memory",
         "tables": [],
     }.items() <= response.json().items()
+
+
+# Shaped after datasette-litestream's startup hook, which schedules a
+# background task with asyncio.get_running_loop().create_task(...):
+# https://github.com/datasette/datasette-litestream
+MARKER_TASK_PLUGIN = """
+import asyncio
+from datasette import hookimpl
+from datasette.utils.asgi import Response
+
+
+@hookimpl
+def startup(datasette):
+    datasette._startup_calls = getattr(datasette, "_startup_calls", 0) + 1
+
+    async def _mark():
+        # Must await before setting the flag: a task with no internal
+        # await point could finish on the throwaway loop before it
+        # closed, masking the regression this test guards against.
+        await asyncio.sleep(0.2)
+        datasette._marker_task_ran = True
+
+    asyncio.get_running_loop().create_task(_mark())
+
+
+@hookimpl
+def register_routes():
+    async def marker_status(datasette):
+        return Response.json(
+            {
+                "marker_task_ran": getattr(datasette, "_marker_task_ran", False),
+                "startup_calls": getattr(datasette, "_startup_calls", 0),
+            }
+        )
+
+    return [(r"^/-/marker-task-ran$", marker_status)]
+"""
+
+
+STARTUP_ERROR_PLUGIN = """
+from datasette import hookimpl
+from datasette.utils import StartupError
+
+
+@hookimpl
+def startup(datasette):
+    raise StartupError("boom from plugin")
+"""
+
+
+@pytest.mark.serial
+def test_startup_hook_background_task_runs_on_serving_loop(serve_with_plugins):
+    """
+    Litestream-shaped regression test: a startup hook that does
+    asyncio.get_running_loop().create_task(...) must have that task
+    actually execute before/while the server is handling requests. This
+    only holds if invoke_startup() and uvicorn.Server.serve() share one
+    event loop. This test fails against unmodified main, where
+    invoke_startup() runs on a throwaway loop that is closed before
+    uvicorn opens its own loop to serve.
+    """
+    _, port = serve_with_plugins({"marker_task_plugin": MARKER_TASK_PLUGIN})
+    # The fixture has already waited for the server to answer requests. The
+    # marker task deliberately awaits before setting its flag, so poll for a
+    # moment rather than assuming it landed before the first request arrived.
+    deadline = time.time() + 3.0
+    payload = {}
+    while time.time() < deadline:
+        payload = httpx2.get(
+            f"http://127.0.0.1:{port}/-/marker-task-ran", timeout=1.0
+        ).json()
+        if payload["marker_task_ran"]:
+            break
+        time.sleep(0.05)
+    assert payload.get("marker_task_ran"), (
+        "The startup hook's asyncio.create_task(...) never ran - "
+        "invoke_startup() and the server are not sharing an event loop"
+    )
+    # Polling above means this test would also pass if the startup hook were
+    # re-run on the serving loop by the first-request fallback - which would
+    # hide exactly the bug being tested. invoke_startup() is idempotent today
+    # so that cannot happen; assert it explicitly so that if the idempotency
+    # guard is ever removed this test fails loudly instead of silently
+    # becoming a no-op.
+    assert payload["startup_calls"] == 1, (
+        "startup hook ran {} times - the marker may have been set by a "
+        "re-run on the serving loop rather than by the original task".format(
+            payload["startup_calls"]
+        )
+    )
+
+
+@pytest.mark.serial
+def test_startup_error_fails_fast_before_port_binds(serve_with_plugins):
+    """
+    A "startup" plugin hook that raises StartupError must fail fast: print
+    the message, exit non-zero, and never accept a connection on the port -
+    the failure must happen before uvicorn.Server binds the socket.
+    """
+    proc, port = serve_with_plugins(
+        {"startup_error_plugin": STARTUP_ERROR_PLUGIN}, wait_for_startup=False
+    )
+    stdout, _ = proc.communicate(timeout=15)
+    output = stdout.decode("utf-8")
+    assert proc.returncode not in (0, None), output
+    assert "boom from plugin" in output, output
+
+    # Nothing is listening on the port now the process has exited. This
+    # confirms the socket was not left bound; on its own it cannot prove the
+    # failure preceded the bind, since a port nothing ever touched also
+    # refuses connections.
+    with (
+        pytest.raises(OSError),
+        socket.create_connection(("127.0.0.1", port), timeout=0.2),
+    ):
+        pass
+
+
+# Verify that SIGTERM and SIGINT sent to `datasette serve` trigger uvicorn's
+# lifespan.shutdown event and run the plugin shutdown hooks. The plugin below
+# writes a sentinel file from its shutdown hook so the tests can check that
+# cleanup ran after the server subprocess exits.
+SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE = """
+import pathlib
+from datasette import hookimpl
+
+SENTINEL_PATH = {sentinel_path!r}
+
+
+@hookimpl
+def shutdown(datasette):
+    pathlib.Path(SENTINEL_PATH).write_text("shutdown ran", "utf-8")
+"""
+
+
+def _start_serve_with_shutdown_sentinel(serve_with_plugins, tmp_path):
+    sentinel_path = tmp_path / "shutdown-sentinel.txt"
+    proc, _ = serve_with_plugins(
+        {
+            "shutdown_sentinel_plugin": SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE.format(
+                sentinel_path=str(sentinel_path)
+            )
+        }
+    )
+    return proc, sentinel_path
+
+
+@pytest.mark.serial
+def test_sigterm_runs_shutdown_hooks(serve_with_plugins, tmp_path):
+    ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel(
+        serve_with_plugins, tmp_path
+    )
+    assert not sentinel_path.exists()
+    ds_proc.send_signal(signal.SIGTERM)
+    try:
+        ds_proc.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        ds_proc.kill()
+        ds_proc.wait()
+        raise AssertionError(
+            "datasette serve did not exit within 10s of SIGTERM\n"
+            + ds_proc.stdout.read().decode("utf-8")
+        )
+    output = ds_proc.stdout.read().decode("utf-8")
+    assert sentinel_path.exists(), (
+        "shutdown hook never wrote its sentinel file after SIGTERM\n" + output
+    )
+    assert sentinel_path.read_text("utf-8") == "shutdown ran"
+
+
+@pytest.mark.serial
+@pytest.mark.skipif(
+    not hasattr(signal, "SIGINT"), reason="Requires signal.SIGINT support"
+)
+def test_sigint_runs_shutdown_hooks(serve_with_plugins, tmp_path):
+    ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel(
+        serve_with_plugins, tmp_path
+    )
+    assert not sentinel_path.exists()
+    ds_proc.send_signal(signal.SIGINT)
+    try:
+        ds_proc.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        ds_proc.kill()
+        ds_proc.wait()
+        raise AssertionError(
+            "datasette serve did not exit within 10s of SIGINT\n"
+            + ds_proc.stdout.read().decode("utf-8")
+        )
+    output = ds_proc.stdout.read().decode("utf-8")
+    assert sentinel_path.exists(), (
+        "shutdown hook never wrote its sentinel file after SIGINT\n" + output
+    )
+    assert sentinel_path.read_text("utf-8") == "shutdown ran"

tests/test_column_types.py

@@ -31,6 +31,7 @@ def ds_ct(tmp_path_factory):
         "'https://example.com', '{\"key\": \"value\"}')"
     )
     db.commit()
+    db.close()
     ds = Datasette(
         [db_path],
         config={
@@ -70,6 +71,7 @@ def ds_ct_editor_permission(tmp_path_factory):
         "'https://example.com', '{\"key\": \"value\"}')"
     )
     db.commit()
+    db.close()
     ds = Datasette(
         [db_path],
         config={

tests/test_csv.py

@@ -166,6 +166,66 @@ async def test_custom_sql_csv(ds_client):
     assert response.text == EXPECTED_CUSTOM_CSV
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("download", (False, True))
+@pytest.mark.parametrize(
+    "query_string,expected_error",
+    (
+        ("sql=select+blah", "no such column: blah"),
+        ("sql=select+*+from+missing", "no such table: missing"),
+        ("sql=select+from", 'near "from": syntax error'),
+        (
+            "sql=delete+from+simple_primary_key",
+            "Statement must be a SELECT",
+        ),
+        ("", "?sql= is required"),
+        (
+            "sql=select+sleep(0.01)&_timelimit=5",
+            (
+                "SQL query took too long. The time limit is"
+                " controlled by the sql_time_limit_ms setting."
+            ),
+        ),
+    ),
+)
+async def test_custom_sql_csv_errors(ds_client, query_string, expected_error, download):
+    if download:
+        query_string += "&_dl=1"
+    response = await ds_client.get(f"/fixtures/-/query.csv?{query_string}")
+    assert response.status_code == 400
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert "content-disposition" not in response.headers
+    assert response.text == expected_error
+
+
+@pytest.mark.asyncio
+async def test_custom_sql_csv_error_head(ds_client):
+    response = await ds_client.head("/fixtures/-/query.csv?sql=select+blah")
+    assert response.status_code == 400
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.content == b""
+
+
+@pytest.mark.asyncio
+async def test_custom_sql_csv_error_cors():
+    ds = Datasette(cors=True)
+    response = await ds.client.get("/_memory/-/query.csv?sql=select+blah")
+    assert response.status_code == 400
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.headers["access-control-allow-origin"] == "*"
+    assert response.text == "no such column: blah"
+
+
+@pytest.mark.asyncio
+async def test_table_csv_error(ds_client):
+    response = await ds_client.get(
+        "/fixtures/simple_primary_key.csv?_where=blah&_stream=1"
+    )
+    assert response.status_code == 400
+    assert response.headers["content-type"] == "text/plain; charset=utf-8"
+    assert response.text == "no such column: blah"
+
+
 @pytest.mark.asyncio
 async def test_table_csv_download(ds_client):
     response = await ds_client.get("/fixtures/simple_primary_key.csv?_dl=1")
@@ -229,6 +289,20 @@ async def test_table_csv_stream(ds_client):
     assert len([b for b in response.content.split(b"\r\n") if b]) == 1002
 
 
+@pytest.mark.asyncio
+async def test_view_csv_stream(ds_client):
+    # Without _stream should return header + 100 rows:
+    response = await ds_client.get("/fixtures/paginated_view.csv?_size=max")
+    assert len([b for b in response.content.split(b"\r\n") if b]) == 101
+    # With _stream=1 should paginate through all pages and return header + 202 rows
+    response = await ds_client.get("/fixtures/paginated_view.csv?_stream=1")
+    lines = [b for b in response.content.split(b"\r\n") if b]
+    assert len(lines) == 203
+    # Ensure there are no duplicate rows from looping
+    assert len(set(lines[1:])) == 202
+    assert lines[0] == b"content,content_extra"
+
+
 def test_csv_trace(app_client_with_trace):
     response = app_client_with_trace.get("/fixtures/simple_primary_key.csv?_trace=1")
     assert response.headers["content-type"] == "text/html; charset=utf-8"

tests/test_docs.py

@@ -27,14 +27,20 @@ def get_labels(filename):
 
 
 @pytest.fixture(scope="session")
-def settings_headings():
-    return get_headings((docs_path / "settings.rst").read_text(), "~")
+def settings_sections():
+    content = (docs_path / "settings.rst").read_text()
+    sections = re.split(r"^(\w+)\n~+\n", content, flags=re.MULTILINE)
+    return dict(zip(sections[1::2], sections[2::2]))
 
 
-def test_settings_are_documented(settings_headings, subtests):
+def test_settings_are_documented(settings_sections, subtests):
     for setting in app.SETTINGS:
         with subtests.test(setting=setting.name):
-            assert setting.name in settings_headings
+            assert setting.name in settings_sections
+            assert (
+                f'setting_default(cog, "{setting.name}")'
+                in settings_sections[setting.name]
+            )
 
 
 @pytest.fixture(scope="session")

tests/test_facets.py

@@ -1,4 +1,5 @@
 import json
+from urllib.parse import parse_qsl, urlsplit
 
 import pytest
 
@@ -151,6 +152,63 @@ async def test_column_facet_results(ds_client):
     ] == buckets
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "column,filters,remaining",
+    [
+        ("COUNTY", "COUNTY=Lee", []),
+        ("COUNTY", "COUNTY__exact=Lee", []),
+        ("COUNTY", "COUNTY=Lee&COUNTY__exact=Lee", []),
+        ("COUNTY", "COUNTY__exact=Lee&COUNTY__exact=Lee", []),
+        (
+            "COUNTY",
+            "COUNTY__exact=Lee&COUNTY__exact=Polk",
+            [("COUNTY__exact", "Polk")],
+        ),
+        ("_county", "_county__exact=Lee", []),
+        ("_county", "_county__exact=Lee&_county=Lee", [("_county", "Lee")]),
+    ],
+)
+async def test_column_facet_selected_exact_filters(
+    ds_client, column, filters, remaining
+):
+    facet = ColumnFacet(
+        ds_client.ds,
+        Request.fake(f"/?_facet={column}&{filters}&other=keep&_sort={column}"),
+        database="fixtures",
+        sql=f"select 'Lee' as {column}",
+    )
+    buckets, timed_out = await facet.facet_results()
+    assert not timed_out
+    result = buckets[0]["results"][0]
+    assert result["selected"] is True
+    assert parse_qsl(urlsplit(result["toggle_url"]).query) == [
+        ("_facet", column),
+        *remaining,
+        ("other", "keep"),
+        ("_sort", column),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_column_facet_underscore_argument_is_not_a_filter(ds_client):
+    facet = ColumnFacet(
+        ds_client.ds,
+        Request.fake("/?_facet=_county&_county=Lee"),
+        database="fixtures",
+        sql="select 'Lee' as _county",
+    )
+    buckets, timed_out = await facet.facet_results()
+    assert not timed_out
+    result = buckets[0]["results"][0]
+    assert result["selected"] is False
+    assert parse_qsl(urlsplit(result["toggle_url"]).query) == [
+        ("_facet", "_county"),
+        ("_county", "Lee"),
+        ("_county__exact", "Lee"),
+    ]
+
+
 @pytest.mark.asyncio
 async def test_column_facet_results_column_starts_with_underscore(ds_client):
     facet = ColumnFacet(

tests/test_fts_permissions.py

@@ -0,0 +1,412 @@
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.permissions import Action, PermissionSQL, _permission_check_cache
+from datasette.resources import DatabaseResource, TableResource
+from datasette.utils.sqlite import sqlite3, sqlite_derived_table_dependencies
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ["fts4", "fts5"])
+@pytest.mark.parametrize("actor", [None, {"id": "root"}], ids=["anonymous", "root"])
+async def test_derived_permissions_allow_one_hop_but_deny_nested_sources(
+    fts_module, actor
+):
+    class InspectPlugin:
+        @hookimpl
+        def register_actions(self):
+            return [
+                Action(
+                    name="inspect-derived",
+                    description="Inspect a table",
+                    resource_class=TableResource,
+                    also_requires="view-table",
+                )
+            ]
+
+        @hookimpl
+        def permission_resources_sql(self, action):
+            if action == "inspect-derived":
+                return PermissionSQL(
+                    sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'inspect allowed' AS reason"
+                )
+
+    ds = Datasette(memory=True)
+    ds.pm.register(InspectPlugin(), name="inspect-derived-test")
+    db = ds.add_memory_database(
+        f"derived_one_hop_{fts_module}_{actor is not None}", name="data"
+    )
+    await db.execute_write("create table Documents (body text)")
+    await db.execute_write(
+        f"create virtual table Search using {fts_module}(body, content='Documents')"
+    )
+    await db.execute_write(
+        f"create virtual table Nested using {fts_module}(body, content='sEaRcH')"
+    )
+    await ds.invoke_startup()
+    token = _permission_check_cache.set({})
+    try:
+        # Both direct permissions are allowed, but a derived source makes its
+        # dependent unavailable even to an actor who can view the whole chain.
+        # Check and cache Search first so its cached grant cannot grant Nested.
+        for table, expected in (
+            ("Documents", True),
+            ("Search", True),
+            ("Nested", False),
+            ("Search_docsize", False),
+        ):
+            for spelling in (table, table.upper(), table.lower()):
+                assert await ds.allowed_many(
+                    actions=["view-table", "inspect-derived"],
+                    resource=TableResource("data", spelling),
+                    actor=actor,
+                ) == {"view-table": expected, "inspect-derived": expected}
+
+        page = await ds.allowed_resources(
+            "view-table", actor, parent="data", include_is_private=True, limit=1000
+        )
+        allowed = {resource.child for resource in page.resources}
+        assert {"Documents", "Search"}.issubset(allowed)
+        assert "Nested" not in allowed
+        assert "Search_docsize" not in allowed
+    finally:
+        _permission_check_cache.reset(token)
+        ds.pm.unregister(name="inspect-derived-test")
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("listing", [False, True], ids=["individual", "listing"])
+async def test_derived_permission_discovery_error_is_retried(monkeypatch, listing):
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(f"derived_discovery_error_{listing}", name="data")
+    await db.execute_write("create table documents (id integer primary key)")
+    await ds.invoke_startup()
+
+    class UnavailableSchema:
+        def execute(self, sql):
+            raise sqlite3.DatabaseError("schema temporarily unavailable")
+
+    async def check():
+        if listing:
+            return await ds.allowed_resources("view-table", parent="data")
+        return await ds.allowed(
+            action="view-table", resource=TableResource("data", "documents")
+        )
+
+    token = _permission_check_cache.set({})
+    try:
+        with monkeypatch.context() as patch:
+            patch.setattr(
+                "datasette.database.sqlite_derived_table_dependencies",
+                lambda conn: sqlite_derived_table_dependencies(UnavailableSchema()),
+            )
+            with pytest.raises(sqlite3.DatabaseError, match="schema temporarily"):
+                await check()
+
+        # Failed discovery must not cache an empty map or a permission grant.
+        assert db._cached_derived_table_dependencies is None
+        assert not _permission_check_cache.get()
+        result = await check()
+        if listing:
+            assert [resource.child for resource in result.resources] == ["documents"]
+        else:
+            assert result is True
+        assert db._cached_derived_table_dependencies is not None
+    finally:
+        _permission_check_cache.reset(token)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ("fts4", "fts5"))
+async def test_external_content_fts_inherits_content_table_view_permission(fts_module):
+    actor = {"id": "reader"}
+    secret_marker = "ISSUE_17_EXTERNAL_CONTENT_FTS_SECRET"
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": {"id": "reader"},
+                "view-database": {"id": "reader"},
+                "view-table": {"id": "reader"},
+                "execute-sql": {"id": "nobody"},
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": False}},
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database(f"issue_17_{fts_module}_permissions", name="data")
+    await db.execute_write("create table secret (id integer primary key, body text)")
+    await db.execute_write(
+        "insert into secret (body) values (?)",
+        [secret_marker],
+    )
+    fts_options = "body, content='secret'"
+    if fts_module == "fts5":
+        fts_options += ", content_rowid='id'"
+    await db.execute_write(
+        f"create virtual table secret_fts using {fts_module}({fts_options})"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        assert "secret_fts" in await db.hidden_table_names()
+        assert (
+            await ds.allowed(
+                action="execute-sql",
+                resource=DatabaseResource("data"),
+                actor=actor,
+            )
+            is False
+        )
+
+        direct = await ds.client.get("/data/secret.json", actor=actor)
+        assert direct.status_code == 403
+
+        companion = await ds.client.get(
+            "/data/secret_fts.json?_shape=array",
+            actor=actor,
+        )
+        assert companion.status_code in (403, 404), (
+            "An automatically hidden external-content FTS table must inherit "
+            "the content table's view denial or be unavailable: "
+            f"{companion.text}"
+        )
+        assert secret_marker not in companion.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ("fts4", "fts5"))
+@pytest.mark.parametrize("contentless", (False, True), ids=("internal", "contentless"))
+async def test_fts_shadow_tables_inherit_logical_table_view_permission(
+    fts_module, contentless
+):
+    table_config = {
+        "secret_fts": {"permissions": {"view-table": False}},
+        # An explicit allow on one implementation table must not override
+        # the logical FTS table's denial.
+        "secret_fts_docsize": {"permissions": {"view-table": True}},
+    }
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+                "view-table": True,
+                "execute-sql": False,
+            },
+            "databases": {"data": {"tables": table_config}},
+        },
+    )
+    db = ds.add_memory_database(
+        f"issue_17_{fts_module}_{'contentless' if contentless else 'internal'}",
+        name="data",
+    )
+    options = "body, content=''" if contentless else "body"
+    await db.execute_write(
+        f"create virtual table secret_fts using {fts_module}({options})"
+    )
+    await db.execute_write(
+        "insert into secret_fts(rowid, body) values (1, 'ISSUE_17_SHADOW_SECRET')"
+    )
+    await ds.invoke_startup()
+
+    try:
+        dependencies = await db.derived_table_dependencies()
+        shadow_tables = sorted(
+            table for table, source in dependencies.items() if source == "secret_fts"
+        )
+        assert shadow_tables
+        assert "secret_fts_docsize" in shadow_tables
+
+        for shadow_table in shadow_tables:
+            assert (
+                await ds.allowed(
+                    action="view-table",
+                    resource=TableResource("data", shadow_table),
+                )
+                is False
+            )
+            response = await ds.client.get(f"/data/{shadow_table}.json?_shape=array")
+            assert response.status_code == 403
+            assert "ISSUE_17_SHADOW_SECRET" not in response.text
+
+        allowed = await ds.allowed_resources("view-table", parent="data", limit=1000)
+        allowed_names = {resource.child for resource in allowed.resources}
+        assert not set(shadow_tables).intersection(allowed_names)
+
+        database_json = await ds.client.get("/data.json")
+        assert database_json.status_code == 200
+        for shadow_table in shadow_tables:
+            assert shadow_table not in database_json.text
+
+        schema_json = await ds.client.get("/data/-/schema.json")
+        assert schema_json.status_code == 200
+        for shadow_table in shadow_tables:
+            assert shadow_table not in schema_json.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "content_allowed,companion_allowed,expected",
+    (
+        (False, True, False),
+        (True, False, False),
+        (True, True, True),
+    ),
+)
+async def test_external_content_and_companion_permissions_are_both_required(
+    content_allowed, companion_allowed, expected
+):
+    ds = Datasette(
+        memory=True,
+        default_deny=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": content_allowed}},
+                        "secret_fts": {
+                            "permissions": {"view-table": companion_allowed}
+                        },
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database(
+        f"issue_17_explicit_{int(content_allowed)}_{int(companion_allowed)}",
+        name="data",
+    )
+    await db.execute_write("create table secret(id integer primary key, body text)")
+    await db.execute_write("insert into secret(body) values ('ISSUE_17_MATRIX_SECRET')")
+    await db.execute_write(
+        "create virtual table secret_fts using fts5("
+        "body, content='secret', content_rowid='id')"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        assert (
+            await ds.allowed(
+                action="view-table",
+                resource=TableResource("data", "secret_fts"),
+            )
+            is expected
+        )
+        response = await ds.client.get("/data/secret_fts.json?_shape=array")
+        assert response.status_code == (200 if expected else 403)
+        if not expected:
+            assert "ISSUE_17_MATRIX_SECRET" not in response.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_derived_tables_propagate_private_flag_and_route_permissions():
+    actor = {"id": "reader"}
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+                "view-table": True,
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": {"id": "reader"}}}
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database("issue_17_private_flag", name="data")
+    await db.execute_write("create table secret(id integer primary key, body text)")
+    await db.execute_write("insert into secret(body) values ('PRIVATE')")
+    await db.execute_write(
+        "create virtual table secret_fts using fts5("
+        "body, content='secret', content_rowid='id')"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        actor_page = await ds.allowed_resources(
+            "view-table", actor, parent="data", include_is_private=True, limit=1000
+        )
+        actor_resources = {
+            resource.child: resource for resource in actor_page.resources
+        }
+        derived_names = set(await db.derived_table_dependencies())
+        assert "secret_fts" in actor_resources
+        assert actor_resources["secret_fts"].private
+        # Shadow tables depend on the already-derived external-content FTS
+        # table, so they remain unavailable even to the permitted reader.
+        assert not (derived_names - {"secret_fts"}).intersection(actor_resources)
+
+        anonymous_page = await ds.allowed_resources(
+            "view-table", parent="data", limit=1000
+        )
+        anonymous_names = {resource.child for resource in anonymous_page.resources}
+        assert not derived_names.intersection(anonymous_names)
+
+        for path in (
+            "/data/secret_fts.json?_facet=body",
+            "/data/secret_fts.csv",
+            "/data/secret_fts/-/autocomplete?q=PRIVATE",
+            "/data/secret_fts/-/schema.json",
+        ):
+            denied = await ds.client.get(path)
+            assert denied.status_code == 403
+            allowed = await ds.client.get(path, actor=actor)
+            assert allowed.status_code == 200
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_cyclic_derived_table_dependencies_fail_closed():
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database("issue_17_cycle", name="data")
+    await db.execute_write(
+        "create virtual table first_fts using fts5(body, content='second_fts')"
+    )
+    await db.execute_write(
+        "create virtual table second_fts using fts5(body, content='first_fts')"
+    )
+    await ds.invoke_startup()
+
+    try:
+        for table in ("first_fts", "second_fts"):
+            assert (
+                await ds.allowed(
+                    action="view-table", resource=TableResource("data", table)
+                )
+                is False
+            )
+
+        page = await ds.allowed_resources("view-table", parent="data", limit=1000)
+        allowed_names = {resource.child for resource in page.resources}
+        assert "first_fts" not in allowed_names
+        assert "second_fts" not in allowed_names
+    finally:
+        ds.close()

tests/test_html.py

@@ -36,8 +36,10 @@ def test_homepage(app_client_two_attached_databases):
     h2 = soup.select("h2")[0]
     assert "extra database" == h2.text.strip()
     counts_p, links_p = h2.find_all_next("p")[:2]
+    # Shadow tables of the external-content index are denied, so they do not
+    # contribute to the table or row totals.
     assert (
-        "2 rows in 1 table, 5 rows in 4 hidden tables, 1 view" == counts_p.text.strip()
+        "2 rows in 1 table, 2 rows in 1 hidden table, 1 view" == counts_p.text.strip()
     )
     # We should only show visible, not hidden tables here:
     table_links = [

tests/test_internals_database.py

@@ -15,11 +15,16 @@ from datasette.database import (
     DatasetteClosedError,
     ExecuteWriteResult,
     MultipleValues,
+    QueryInterrupted,
     Results,
     _deliver_write_result,
 )
 from datasette.utils import Column
-from datasette.utils.sqlite import sqlite3, supports_returning
+from datasette.utils.sqlite import (
+    sqlite3,
+    sqlite_derived_table_dependencies,
+    supports_returning,
+)
 
 requires_sqlite_returning = pytest.mark.skipif(
     not supports_returning(), reason="SQLite does not support RETURNING"
@@ -38,6 +43,31 @@ async def test_execute(db):
     assert 15 == len(results)
 
 
+@pytest.mark.asyncio
+async def test_derived_dependency_cache_survives_failed_refresh(monkeypatch):
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.derived_table_dependencies()
+    previous_cache = db._cached_derived_table_dependencies
+    await db.execute_write("create table dependency_cache_refresh (id integer)")
+
+    class UnavailableSchema:
+        def execute(self, sql):
+            raise sqlite3.DatabaseError("schema temporarily unavailable")
+
+    with monkeypatch.context() as patch:
+        patch.setattr(
+            "datasette.database.sqlite_derived_table_dependencies",
+            lambda conn: sqlite_derived_table_dependencies(UnavailableSchema()),
+        )
+        with pytest.raises(sqlite3.DatabaseError, match="schema temporarily"):
+            await db.derived_table_dependencies()
+    assert db._cached_derived_table_dependencies == previous_cache
+
+    await db.derived_table_dependencies()
+    assert db._cached_derived_table_dependencies[0] != previous_cache[0]
+
+
 @pytest.mark.asyncio
 async def test_results_first(db):
     assert None is (await db.execute("select * from facetable where pk > 100")).first()
@@ -478,6 +508,31 @@ async def test_view_names(db):
     ]
 
 
+@pytest.mark.asyncio
+async def test_execute_write_custom_time_limit():
+    ds = Datasette(settings={"sql_time_limit_ms": 1})
+    db = ds.add_memory_database(uuid.uuid4().hex, name="write_limits")
+    await ds.invoke_startup()
+    # Bounded work from PR #51; even without a limit this finishes on its own.
+    sql = (
+        "with recursive c(x) as "
+        "(select 1 union all select x+1 from c where x < 800000) "
+        "select x from c where x < 0"
+    )
+    try:
+        await db.execute_write("create table items(value integer)")
+        with pytest.raises(QueryInterrupted):
+            await db.execute(sql)
+        # Writes take their own explicit limit, independent of the read setting.
+        with pytest.raises(QueryInterrupted):
+            await db.execute_write(f"insert into items(value) {sql}", time_limit_ms=1)
+        # Interruption must leave the connection available for subsequent writes.
+        await db.execute_write("insert into items(value) values (1)")
+        assert (await db.execute("select value from items")).single_value() == 1
+    finally:
+        ds.close()
+
+
 @pytest.mark.asyncio
 async def test_execute_write_block_true(db):
     result = await db.execute_write(
@@ -705,6 +760,33 @@ async def test_execute_write_fn_block_false(db):
     assert isinstance(task_id, uuid.UUID)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("disable_threads", (False, True))
+async def test_execute_write_fn_block_false_returns_uuid(tmp_path, disable_threads):
+    # block=False is documented to return "a UUID representing the queued task".
+    # With num_sql_threads=0 there is no write thread, so the non-threaded branch
+    # has to satisfy the same contract as the threaded one.
+    settings = {"num_sql_threads": 0} if disable_threads else {}
+    ds = Datasette([], memory=True, settings=settings)
+    await ds.invoke_startup()
+    db = ds.add_memory_database("test_block_false")
+    await db.execute_write(
+        "create table if not exists t (id integer primary key, v text)"
+    )
+
+    def write_fn(conn):
+        conn.execute("insert into t (v) values ('a')")
+        # Returns None, like most write functions.
+
+    task_id = await db.execute_write_fn(write_fn, block=False)
+
+    assert isinstance(task_id, uuid.UUID)
+    # Distinct per call, so a caller can tell two queued tasks apart.
+    second = await db.execute_write_fn(write_fn, block=False)
+    assert isinstance(second, uuid.UUID)
+    assert second != task_id
+
+
 @pytest.mark.asyncio
 async def test_execute_write_fn_block_true(db):
     def write_fn(conn):
@@ -1223,3 +1305,17 @@ async def test_database_close_is_idempotent(tmpdir):
     # Second call should be a no-op, not raise
     db.close()
     ds._internal_database.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("num_sql_threads", [0, 2])
+@pytest.mark.parametrize("named", [False, True])
+async def test_close_releases_memory_connections(num_sql_threads, named):
+    ds = Datasette(memory=True, settings={"num_sql_threads": num_sql_threads})
+    db = ds.add_memory_database(uuid.uuid4().hex) if named else ds.get_database()
+    read_connection = await db.execute_fn(lambda conn: conn)
+    write_connection = await db.execute_write_fn(lambda conn: conn)
+    ds.close()
+    for conn in (read_connection, write_connection):
+        with pytest.raises(sqlite3.ProgrammingError, match="closed"):
+            conn.execute("select 1")

tests/test_internals_datasette_client.py

@@ -1,4 +1,4 @@
-import httpx
+import httpx2
 import pytest
 import pytest_asyncio
 
@@ -43,7 +43,7 @@ async def datasette_with_permissions():
 async def test_client_methods(datasette, method, path, expected_status):
     client_method = getattr(datasette.client, method)
     response = await client_method(path)
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
     assert response.status_code == expected_status
     # Try that again using datasette.client.request
     response2 = await datasette.client.request(method, path)
@@ -63,7 +63,7 @@ async def test_client_post(datasette, prefix):
                 "message": "A message",
             },
         )
-        assert isinstance(response, httpx.Response)
+        assert isinstance(response, httpx2.Response)
         assert response.status_code == 302
         assert "ds_messages" in response.cookies
     finally:
@@ -135,7 +135,7 @@ async def test_skip_permission_checks_all_methods(datasette_with_permissions, me
     response = await client_method("/test_db.json", skip_permission_checks=True)
     # We don't check status code since some methods might not be allowed,
     # but we verify the request doesn't fail due to permissions
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
 
 
 @pytest.mark.asyncio
@@ -340,7 +340,7 @@ async def test_actor_parameter_all_http_methods(datasette, method):
     client_method = getattr(datasette.client, method)
     # Just verify no TypeError about unexpected 'actor' kwarg
     response = await client_method("/", actor={"id": "root"})
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
 
 
 @pytest.mark.asyncio

tests/test_internals_request.py

@@ -35,6 +35,51 @@ def _receive_chunks(chunks):
     return receive
 
 
+@pytest.mark.parametrize(
+    "header_name", [b"content-type", b"Content-Type", b"CONTENT-TYPE"]
+)
+@pytest.mark.parametrize("lookup", ["content-type", "Content-Type", "CONTENT-TYPE"])
+def test_request_headers_case_insensitive(header_name, lookup):
+    request = Request({"headers": [(header_name, b"application/json")]}, None)
+    assert request.headers.get(lookup) == "application/json"
+    assert request.headers[lookup] == "application/json"
+    assert lookup in request.headers
+
+
+def test_request_headers_mapping():
+    request = Request(
+        {
+            "headers": [
+                (b"Content-Type", b"application/json"),
+                (b"X-Title", "café".encode("latin-1")),
+                (b"CONTENT-TYPE", b"text/plain"),
+            ]
+        },
+        None,
+    )
+    headers = request.headers
+    expected = {"content-type": "text/plain", "x-title": "café"}
+    assert headers == expected
+    assert dict(headers) == expected
+    assert list(headers) == list(expected)
+    assert list(headers.keys()) == list(expected.keys())
+    assert list(headers.items()) == list(expected.items())
+    assert json.loads(json.dumps(headers)) == expected
+    assert headers["Content-Type"] == "text/plain"
+    assert headers["X-Title"] == "café"
+
+
+@pytest.mark.parametrize("scope", [{}, {"headers": None}, {"headers": []}])
+def test_request_headers_missing(scope):
+    headers = Request(scope, None).headers
+    assert headers == {}
+    assert headers.get("Content-Type") is None
+    assert headers.get("Content-Type", "default") == "default"
+    assert "Content-Type" not in headers
+    with pytest.raises(KeyError):
+        headers["Content-Type"]
+
+
 @pytest.mark.asyncio
 async def test_request_post_vars():
     scope = {

tests/test_lifespan.py

@@ -0,0 +1,344 @@
+"""
+Tests for wiring Datasette startup (setup_db table counts + invoke_startup)
+into the ASGI lifespan protocol.
+
+These exercise Datasette._startup_sequence() via three different callers:
+- AsgiLifespan, by hand-driving lifespan.startup messages (no HTTP request)
+- AsgiRunOnFirstRequest, the fallback for hosts that never send lifespan
+  events (this is what DatasetteClient / plain httpx2.ASGITransport uses)
+- Both at once, to prove startup hooks run at most once
+"""
+
+import asyncio
+import contextlib
+import sqlite3
+
+import httpx2
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.database import Database
+from datasette.plugins import pm
+
+
+async def _drive_lifespan_startup(app):
+    """Send a single lifespan.startup message into app's ASGI lifespan loop
+    and return the list of messages sent back - without ever sending
+    lifespan.shutdown. Mirrors what a real server does: after startup
+    completes it parks waiting for the next event. We cancel that wait
+    once we've observed the startup response, rather than closing the
+    Datasette instance down with a shutdown message.
+    """
+    messages_sent = []
+    startup_responded = asyncio.Event()
+    delivered = False
+
+    async def receive():
+        nonlocal delivered
+        if not delivered:
+            delivered = True
+            return {"type": "lifespan.startup"}
+        # No further messages: block until the task is cancelled below,
+        # same as a real server parked waiting for lifespan.shutdown.
+        await asyncio.Event().wait()
+
+    async def send(message):
+        messages_sent.append(message)
+        startup_responded.set()
+
+    task = asyncio.create_task(app({"type": "lifespan"}, receive, send))
+    try:
+        await asyncio.wait_for(startup_responded.wait(), timeout=5)
+    finally:
+        task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await task
+    return messages_sent
+
+
+@pytest.mark.asyncio
+async def test_lifespan_startup_runs_before_any_request():
+    ds = Datasette(memory=True)
+    assert ds._startup_invoked is False
+    app = ds.app()
+
+    messages = await _drive_lifespan_startup(app)
+
+    assert {"type": "lifespan.startup.complete"} in messages
+    assert ds._startup_invoked is True
+    # Internal catalog tables should be populated too, entirely without an
+    # HTTP request having been made.
+    internal_db = ds.get_internal_database()
+    databases = await internal_db.execute("select * from catalog_databases")
+    assert len(databases.rows) >= 1
+
+
+@pytest.mark.asyncio
+async def test_lifespan_startup_failure_reports_lifespan_startup_failed():
+    class RaisingStartupPlugin:
+        __name__ = "RaisingStartupPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                raise RuntimeError("boom from startup hook")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(RaisingStartupPlugin(), name="raising_startup_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan_startup(app)
+    finally:
+        pm.unregister(name="raising_startup_plugin")
+
+    assert messages == [
+        {"type": "lifespan.startup.failed", "message": "boom from startup hook"}
+    ]
+    # The exception happened before invoke_startup() got to the end of its
+    # body, so startup is not considered to have completed.
+    assert ds._startup_invoked is False
+
+
+@pytest.mark.asyncio
+async def test_startup_runs_exactly_once_across_lifespan_and_first_request():
+    call_count = {"n": 0}
+
+    class CountingStartupPlugin:
+        __name__ = "CountingStartupPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                call_count["n"] += 1
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(CountingStartupPlugin(), name="counting_startup_plugin")
+    try:
+        # Build the ASGI app once, the way a real deployment does - and
+        # reuse the SAME app instance for both the lifespan drive and the
+        # HTTP requests below, since a fresh ds.app() call would reset the
+        # AsgiRunOnFirstRequest fallback's state.
+        app = ds.app()
+
+        messages = await _drive_lifespan_startup(app)
+        assert {"type": "lifespan.startup.complete"} in messages
+        assert call_count["n"] == 1
+
+        # A first HTTP request (as if the host never sent lifespan events,
+        # or lifespan already ran) should not run the hook again.
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            response1 = await client.get("/-/versions.json")
+            assert response1.status_code == 200
+            # ... nor should a second, repeat request.
+            response2 = await client.get("/-/versions.json")
+            assert response2.status_code == 200
+    finally:
+        pm.unregister(name="counting_startup_plugin")
+
+    assert call_count["n"] == 1
+
+
+@pytest.mark.asyncio
+async def test_no_lifespan_first_request_still_triggers_startup():
+    # Pin today's behavior: a client that never drives ASGI lifespan events
+    # at all (like httpx2.ASGITransport, which DatasetteClient uses) still
+    # gets startup armed by the AsgiRunOnFirstRequest fallback.
+    ds = Datasette(memory=True)
+    assert ds._startup_invoked is False
+    app = ds.app()
+    transport = httpx2.ASGITransport(app=app)
+    async with httpx2.AsyncClient(
+        transport=transport, base_url="http://localhost"
+    ) as client:
+        response = await client.get("/-/versions.json")
+        assert response.status_code == 200
+
+    assert ds._startup_invoked is True
+    internal_db = ds.get_internal_database()
+    databases = await internal_db.execute("select * from catalog_databases")
+    assert len(databases.rows) >= 1
+
+
+@pytest.mark.asyncio
+async def test_datasette_client_first_request_triggers_startup():
+    # Same as above, but through the real DatasetteClient (ds.client) that
+    # plugins and tests actually use, to confirm nothing regressed there.
+    ds = Datasette(memory=True)
+    assert ds._startup_invoked is False
+    response = await ds.client.get("/-/versions.json")
+    assert response.status_code == 200
+    assert ds._startup_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_concurrent_first_requests_all_wait_for_slow_startup():
+    call_count = {"n": 0}
+
+    class SlowStartupPlugin:
+        __name__ = "SlowStartupPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                call_count["n"] += 1
+                await asyncio.sleep(0.2)
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(SlowStartupPlugin(), name="slow_startup_plugin")
+    try:
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            responses = await asyncio.gather(
+                *[client.get("/-/versions.json") for _ in range(10)]
+            )
+    finally:
+        pm.unregister(name="slow_startup_plugin")
+
+    # Every one of the 10 simultaneous first requests must have blocked
+    # until startup actually finished, not raced ahead of it.
+    assert all(response.status_code == 200 for response in responses)
+    assert call_count["n"] == 1
+    assert ds._startup_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_setup_db_still_runs_when_invoke_startup_ran_first(tmp_path, monkeypatch):
+    # Regression test: `datasette serve` (cli.py _serve_async) calls
+    # ds.invoke_startup() directly, before uvicorn ever sends a
+    # lifespan.startup event that drives _startup_sequence(). If
+    # _startup_sequence()'s fast path only checked `_startup_invoked`, it
+    # would see startup already done and skip the immutable-database
+    # table-count precompute (setup_db) entirely - a silent regression
+    # versus main, where AsgiRunOnFirstRequest ran setup_db unconditionally
+    # on request #1.
+    db_path = tmp_path / "immutable.db"
+    conn = sqlite3.connect(str(db_path))
+    conn.execute("create table t (id integer primary key)")
+    conn.commit()
+    conn.close()
+
+    ds = Datasette([], immutables=[str(db_path)])
+
+    call_count = {"n": 0}
+    original_table_counts = Database.table_counts
+
+    async def counting_table_counts(self, *args, **kwargs):
+        call_count["n"] += 1
+        return await original_table_counts(self, *args, **kwargs)
+
+    monkeypatch.setattr(Database, "table_counts", counting_table_counts)
+
+    # Simulate the CLI path: invoke_startup() runs directly and completes
+    # BEFORE _startup_sequence() ever gets a chance to run setup_db.
+    await ds.invoke_startup()
+    assert ds._startup_invoked is True
+    assert call_count["n"] == 0
+
+    # The lifespan/first-request path (or the CLI itself, per the fix)
+    # calling the shared entry point afterwards must still precompute
+    # table counts for immutable databases.
+    await ds._startup_sequence()
+    assert call_count["n"] == 1
+    assert ds._setup_db_done is True
+
+    # Idempotency: a second call must not recompute.
+    await ds._startup_sequence()
+    assert call_count["n"] == 1
+
+
+@pytest.mark.asyncio
+async def test_asgi_wrapper_runs_after_startup_fallback_path():
+    class AssertStartupPlugin:
+        __name__ = "AssertStartupPlugin"
+
+        @hookimpl
+        def asgi_wrapper(self, datasette):
+            def wrap(app):
+                async def check_startup(scope, receive, send):
+                    if scope["type"] == "http":
+                        assert (
+                            datasette._startup_invoked is True
+                        ), "asgi_wrapper saw an http scope before startup completed"
+                    await app(scope, receive, send)
+
+                return check_startup
+
+            return wrap
+
+    ds = Datasette(memory=True)
+    pm.register(AssertStartupPlugin(), name="assert_startup_plugin")
+    try:
+        assert ds._startup_invoked is False
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            response = await client.get("/-/versions.json")
+            assert response.status_code == 200
+    finally:
+        pm.unregister(name="assert_startup_plugin")
+
+    assert ds._startup_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_short_circuit_wrapper_no_longer_defers_startup():
+    # Middleware that returns a response before getting to the rest of
+    # Datasette should still cause _startup_invoked=True
+    class ShortCircuitPlugin:
+        __name__ = "ShortCircuitPlugin"
+
+        @hookimpl
+        def asgi_wrapper(self, datasette):
+            def wrap(app):
+                async def forbidden(scope, receive, send):
+                    if scope["type"] != "http":
+                        await app(scope, receive, send)
+                        return
+                    await send(
+                        {
+                            "type": "http.response.start",
+                            "status": 403,
+                            "headers": [[b"content-type", b"text/plain"]],
+                        }
+                    )
+                    await send(
+                        {
+                            "type": "http.response.body",
+                            "body": b"Forbidden",
+                        }
+                    )
+
+                return forbidden
+
+            return wrap
+
+    ds = Datasette(memory=True)
+    pm.register(ShortCircuitPlugin(), name="short_circuit_plugin")
+    try:
+        assert ds._startup_invoked is False
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            response = await client.get("/-/versions.json")
+            assert response.status_code == 403
+    finally:
+        pm.unregister(name="short_circuit_plugin")
+
+    assert ds._startup_invoked is True

tests/test_load_extensions.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from unittest import mock
 
 import pytest
 
@@ -20,6 +21,29 @@ def has_compiled_ext():
     return False
 
 
+@pytest.mark.parametrize("load_fails", (False, True))
+def test_load_extension_is_disabled(load_fails):
+    ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
+    connection = mock.Mock()
+    if load_fails:
+        connection.load_extension.side_effect = RuntimeError
+
+    if load_fails:
+        with pytest.raises(RuntimeError):
+            ds._prepare_connection(connection, "data")
+    else:
+        ds._prepare_connection(connection, "data")
+
+    # Extensions are loaded using the Python API, never via SQL
+    assert connection.load_extension.mock_calls == [
+        mock.call(COMPILED_EXTENSION_PATH),
+    ]
+    assert connection.enable_load_extension.mock_calls == [
+        mock.call(True),
+        mock.call(False),
+    ]
+
+
 @pytest.mark.asyncio
 @pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
 async def test_load_extension_default_entrypoint():
@@ -64,3 +88,20 @@ async def test_load_extension_multiple_entrypoints():
     response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+c()")
     assert response.status_code == 200
     assert response.json()["rows"][0][0] == "c"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
+async def test_sql_cannot_load_additional_extension():
+    ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
+
+    response = await ds.client.get(
+        "/_memory/-/query.json",
+        params={
+            "sql": "select load_extension(:path, :entrypoint)",
+            "path": COMPILED_EXTENSION_PATH,
+            "entrypoint": "sqlite3_ext_b_init",
+        },
+    )
+    assert response.status_code == 400
+    assert response.json()["error"] == "not authorized"

tests/test_multipart.py

@@ -4,14 +4,145 @@ Tests for request.form() multipart form data parsing.
 Uses TDD approach - these tests are written first, then implementation follows.
 """
 
+import asyncio
 import base64
 import json
+import threading
 from collections import namedtuple
 
 import pytest
 from multipart_form_data_conformance import get_tests_dir
 
 from datasette.utils.asgi import BadRequest, Request
+from datasette.utils.multipart import MultipartParseError, parse_form_data
+
+
+@pytest.fixture
+def upload_files(monkeypatch):
+    from datasette.utils import multipart
+
+    files = []
+    original = multipart.tempfile.SpooledTemporaryFile
+
+    def create_file(*args, **kwargs):
+        file = original(*args, **kwargs)
+        files.append(file)
+        return file
+
+    monkeypatch.setattr(multipart.tempfile, "SpooledTemporaryFile", create_file)
+    yield files
+    for file in files:
+        file.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "failure",
+    [
+        "file_limit",
+        "request_limit",
+        "truncated",
+        "disconnect",
+        "receive_error",
+        "cancel",
+    ],
+)
+async def test_failed_upload_closes_completed_and_partial_files(upload_files, failure):
+    # Complete one file and begin another, flushing the parser's 64 KiB batch.
+    body = (
+        b'--boundary\r\nContent-Disposition: form-data; name="one"; filename="one"\r\n\r\n'
+        b'first\r\n--boundary\r\nContent-Disposition: form-data; name="two"; filename="two"\r\n\r\n'
+        + b"x" * (64 * 1024)
+    )
+    calls = 0
+
+    async def receive():
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            return {"type": "http.request", "body": body, "more_body": True}
+        if failure == "disconnect":
+            return {"type": "http.disconnect"}
+        if failure == "receive_error":
+            raise OSError("receive failed")
+        if failure == "cancel":
+            raise asyncio.CancelledError
+        return {"type": "http.request", "body": b"x" * 100, "more_body": False}
+
+    kwargs = {}
+    if failure == "file_limit":
+        kwargs["max_file_size"] = 1024
+    if failure == "request_limit":
+        kwargs["max_request_size"] = len(body)
+    error = {
+        "receive_error": OSError,
+        "cancel": asyncio.CancelledError,
+    }.get(failure, MultipartParseError)
+    with pytest.raises(error):
+        await parse_form_data(
+            receive, "multipart/form-data; boundary=boundary", files=True, **kwargs
+        )
+    assert len(upload_files) == 2
+    assert all(file.closed for file in upload_files)
+
+
+@pytest.mark.asyncio
+async def test_unnamed_upload_is_closed(upload_files):
+    body = (
+        b'--boundary\r\nContent-Disposition: form-data; filename="ignored"\r\n\r\n'
+        b"content\r\n--boundary--\r\n"
+    )
+    form = await parse_form_data(
+        make_receive(body), "multipart/form-data; boundary=boundary", files=True
+    )
+    assert len(form) == 0
+    assert len(upload_files) == 1
+    assert upload_files[0].closed
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("worker_error", [False, True])
+async def test_cancelled_upload_waits_for_worker(
+    upload_files, monkeypatch, worker_error
+):
+    from datasette.utils.multipart import MultipartParser
+
+    loop = asyncio.get_running_loop()
+    started = asyncio.Event()
+    release = threading.Event()
+    original_feed = MultipartParser.feed
+
+    def blocking_feed(self, chunk):
+        original_feed(self, chunk)
+        loop.call_soon_threadsafe(started.set)
+        assert release.wait(5)
+        if worker_error:
+            raise MultipartParseError("worker failed")
+
+    monkeypatch.setattr(MultipartParser, "feed", blocking_feed)
+    body = (
+        b'--boundary\r\nContent-Disposition: form-data; name="file"; filename="file"\r\n\r\n'
+        + b"x" * (64 * 1024)
+    )
+    task = asyncio.create_task(
+        parse_form_data(
+            make_receive(body), "multipart/form-data; boundary=boundary", files=True
+        )
+    )
+    try:
+        await asyncio.wait_for(started.wait(), timeout=5)
+        # A second cancellation must also leave cleanup waiting for the worker.
+        for _ in range(2):
+            task.cancel()
+            await asyncio.sleep(0)
+        assert not task.done()
+        assert len(upload_files) == 1
+        assert not upload_files[0].closed
+    finally:
+        release.set()
+        with pytest.raises(asyncio.CancelledError):
+            await task
+    assert upload_files[0].closed
 
 
 def make_receive(body: bytes):
@@ -1078,9 +1209,7 @@ async def test_conformance(test_spec, headers, body):
             await request.form(files=True)
         return
 
-    # Parse form data
-    form = await request.form(files=True)
-
+    async with await request.form(files=True) as form:
         # Verify each expected part
         for i, expected_part in enumerate(expected["parts"]):
             name = expected_part["name"]
@@ -1119,9 +1248,9 @@ async def test_conformance(test_spec, headers, body):
                 assert hasattr(value, "filename"), f"Expected file for {name}"
 
                 # Check filename - use filename_star if present, else filename
-            expected_filename = expected_part.get("filename_star") or expected_part.get(
-                "filename"
-            )
+                expected_filename = expected_part.get(
+                    "filename_star"
+                ) or expected_part.get("filename")
                 if expected_filename:
                     assert (
                         value.filename == expected_filename

tests/test_numeric_filter_values.py

@@ -0,0 +1,47 @@
+import sqlite3
+from contextlib import closing
+
+import pytest
+
+from datasette.filters import Filters
+
+
+@pytest.mark.parametrize(
+    "value,expected",
+    (
+        ("3.5", 3.5),
+        ("-2", -2),
+        ("-2.5", -2.5),
+        ("1e3", 1000.0),
+        ("not-a-number", "not-a-number"),
+        ("nan", "nan"),
+        ("inf", "inf"),
+        ("-inf", "-inf"),
+    ),
+)
+def test_numeric_filter_parameters(value, expected):
+    filters = Filters((("score__gt", value),))
+    sql_bits, params = filters.build_where_clauses("items")
+
+    assert sql_bits == ['"score" > :p0']
+    assert params == {"p0": expected}
+
+
+def test_numeric_filter_parameters_against_calculated_view():
+    with closing(sqlite3.connect(":memory:")) as conn:
+        conn.execute("create table searchable(pk integer)")
+        conn.executemany("insert into searchable(pk) values (?)", [(0,), (1,), (2,)])
+        conn.execute(
+            "create view calculated as "
+            "select pk + 1 as pk_plus_one, pk / 2.0 as score from searchable"
+        )
+
+        sql_bits, params = Filters((("score__gt", "0.1"),)).build_where_clauses(
+            "calculated"
+        )
+        rows = conn.execute(
+            "select score from calculated where {}".format(" and ".join(sql_bits)),
+            params,
+        ).fetchall()
+
+        assert rows == [(0.5,), (1.0,)]

tests/test_permission_endpoints.py

@@ -494,3 +494,31 @@ async def test_execute_sql_requires_view_database():
         )
     finally:
         ds.pm.unregister(plugin)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("path", ["/-/allowed", "/-/allowed.json?action=view-table"])
+async def test_allowed_requires_view_instance(path):
+    """
+    GHSA-hp2x-vx2r-6vxg: /-/allowed should be gated like its /-/rules sibling.
+
+    An actor who is denied view-instance gets 403 from / and /-/rules, but
+    /-/allowed (HTML and JSON) currently returns 200 to the same actor.
+    """
+    ds = Datasette(config={"allow": {"id": "alice"}})
+    await ds.invoke_startup()
+    db = ds.add_memory_database("live")
+    await db.execute_write("CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY)")
+    await ds.refresh_schemas()
+
+    assert (await ds.client.get("/")).status_code == 403
+    assert (await ds.client.get("/-/rules.json?action=view-table")).status_code == 403
+
+    response = await ds.client.get(path)
+    assert response.status_code == 403
+
+    # Alice is still allowed
+    response = await ds.client.get(
+        path, cookies={"ds_actor": ds.client.actor_cookie({"id": "alice"})}
+    )
+    assert response.status_code == 200

tests/test_permissions.py

@@ -520,6 +520,7 @@ def view_instance_client():
         "/-/plugins",
         "/-/settings",
         "/-/threads",
+        "/-/tasks",
         "/-/databases",
         "/-/permissions",
         "/-/messages",