Screenshots for reusable modal dialogs docs, plus just shots (#2958)

docs/shots.yml defines documentation screenshots for shot-scraper

"Just shots" runs them.

.github/actions/setup-sqlite-version/action.yml

@@ -0,0 +1,39 @@
+name: "Setup SQLite version"
+description: "Build and activate a specific SQLite version from its amalgamation archive"
+inputs:
+  version:
+    description: "The SQLite version to install"
+    required: true
+  cflags:
+    description: "CFLAGS to use when compiling SQLite"
+    required: false
+    default: ""
+  skip-activate:
+    description: "Set to true to skip modifying the library path"
+    required: false
+    default: "false"
+  fallback-urls:
+    description: "Whitespace-separated fallback download URLs to try after sqlite.org"
+    required: false
+    default: ""
+outputs:
+  sqlite-location:
+    description: "Directory containing the compiled SQLite library"
+    value: ${{ steps.build.outputs.sqlite-location }}
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads"
+    - uses: actions/cache@v6
+      with:
+        path: ${{ runner.temp }}/sqlite-versions/downloads
+        key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1
+    - id: build
+      shell: bash
+      run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh"
+      env:
+        SQLITE_VERSION: ${{ inputs.version }}
+        SQLITE_CFLAGS: ${{ inputs.cflags }}
+        SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }}
+        SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }}

.github/actions/setup-sqlite-version/setup-sqlite-version.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}"
+cflags="${SQLITE_CFLAGS:-}"
+skip_activate="${SQLITE_SKIP_ACTIVATE:-false}"
+extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}"
+
+case "$version_spec" in
+  3.46 | 3.46.0)
+    sqlite_version="3.46.0"
+    sqlite_year="2024"
+    amalgamation_id="3460000"
+    builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip"
+    ;;
+  3.25 | 3.25.0)
+    sqlite_version="3.25.0"
+    sqlite_year="2018"
+    amalgamation_id="3250000"
+    builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3250000.zip?v=1"
+    ;;
+  *)
+    echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh."
+    exit 1
+    ;;
+esac
+
+case "$(uname -s)" in
+  Linux)
+    library_name="libsqlite3.so.0"
+    library_path_var="LD_LIBRARY_PATH"
+    ;;
+  Darwin)
+    library_name="libsqlite3.dylib"
+    library_path_var="DYLD_LIBRARY_PATH"
+    ;;
+  *)
+    echo "::error::Unsupported platform $(uname -s)"
+    exit 1
+    ;;
+esac
+
+runner_temp="${RUNNER_TEMP:-}"
+if [ -z "$runner_temp" ]; then
+  runner_temp="$(mktemp -d)"
+fi
+
+filename="sqlite-amalgamation-${amalgamation_id}"
+official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip"
+download_dir="${runner_temp}/sqlite-versions/downloads"
+source_root="${runner_temp}/sqlite-versions/source"
+source_dir="${source_root}/${filename}"
+build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}"
+archive_path="${download_dir}/${filename}.zip"
+
+mkdir -p "$download_dir" "$source_root" "$build_dir"
+
+download_archive() {
+  local url
+  local candidate_path="${archive_path}.tmp"
+  local urls=("$official_url")
+
+  for url in $builtin_fallback_urls $extra_fallback_urls; do
+    urls+=("$url")
+  done
+
+  rm -f "$candidate_path"
+  for url in "${urls[@]}"; do
+    echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}"
+    if curl \
+      --fail \
+      --location \
+      --show-error \
+      --retry 5 \
+      --retry-delay 2 \
+      --retry-max-time 180 \
+      --retry-all-errors \
+      --connect-timeout 20 \
+      --max-time 240 \
+      --output "$candidate_path" \
+      "$url"; then
+      mv "$candidate_path" "$archive_path"
+      return 0
+    fi
+
+    echo "::warning::Download failed from ${url}"
+    rm -f "$candidate_path"
+  done
+
+  echo "::error::Could not download SQLite ${sqlite_version} amalgamation"
+  return 1
+}
+
+if [ ! -f "${source_dir}/sqlite3.c" ]; then
+  if [ ! -f "$archive_path" ]; then
+    download_archive
+  fi
+
+  rm -rf "$source_dir"
+  unzip -q "$archive_path" -d "$source_root"
+fi
+
+if [ ! -f "${source_dir}/sqlite3.c" ]; then
+  echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}"
+  exit 1
+fi
+
+read -r -a cflag_args <<< "$cflags"
+
+echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}"
+gcc \
+  -fPIC \
+  -shared \
+  "${cflag_args[@]}" \
+  "${source_dir}/sqlite3.c" \
+  "-I${source_dir}" \
+  -o "${build_dir}/${library_name}"
+
+if [ "$library_name" = "libsqlite3.so.0" ]; then
+  ln -sf "$library_name" "${build_dir}/libsqlite3.so"
+fi
+
+if [ -n "${GITHUB_OUTPUT:-}" ]; then
+  echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT"
+else
+  echo "sqlite-location=${build_dir}"
+fi
+
+case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in
+  true | 1 | yes)
+    echo "Skipping ${library_path_var} activation"
+    ;;
+  *)
+    existing_value="${!library_path_var:-}"
+    if [ -n "${GITHUB_ENV:-}" ]; then
+      if [ -n "$existing_value" ]; then
+        echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV"
+      else
+        echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV"
+      fi
+    fi
+    echo "Added ${build_dir} to ${library_path_var}"
+    ;;
+esac

.github/workflows/deploy-branch-preview.yml

@@ -1,35 +0,0 @@
-name: Deploy a Datasette branch preview to Vercel
-
-on:
-  workflow_dispatch:
-    inputs:
-      branch:
-        description: "Branch to deploy"
-        required: true
-        type: string
-
-jobs:
-  deploy-branch-preview:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v6
-      with:
-        python-version: "3.11"
-    - name: Install dependencies
-      run: |
-        pip install datasette-publish-vercel
-    - name: Deploy the preview
-      env:
-        VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }}
-      run: |
-        export BRANCH="${{ github.event.inputs.branch }}"
-        wget https://latest.datasette.io/fixtures.db
-        datasette publish vercel fixtures.db \
-          --branch $BRANCH \
-          --project "datasette-preview-$BRANCH" \
-          --token $VERCEL_TOKEN \
-          --scope datasette \
-          --about "Preview of $BRANCH" \
-          --about_url "https://github.com/simonw/datasette/tree/$BRANCH"

.github/workflows/deploy-latest.yml

@@ -14,25 +14,46 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
+    - name: Check deployment prerequisites
+      id: deployment-prerequisites
+      env:
+        GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+        LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
+      run: |
+        missing=()
+        for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do
+          if [[ -z "${!variable:-}" ]]; then
+            missing+=("$variable")
+          fi
+        done
+        if (( ${#missing[@]} )); then
+          echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}"
+          echo "available=false" >> "$GITHUB_OUTPUT"
+        else
+          echo "available=true" >> "$GITHUB_OUTPUT"
+        fi
     - name: Check out datasette
-      uses: actions/checkout@v5
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
+      uses: actions/checkout@v7
     - name: Set up Python
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/setup-python@v6
       with:
         python-version: "3.13"
         cache: pip
     - name: Install Python dependencies
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -e .[test]
+        python -m pip install . --group dev
-        python -m pip install -e .[docs]
+        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
-        python -m pip install sphinx-to-sqlite==0.1a1
     - name: Run tests
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
     - name: Build fixtures.db and other files needed to deploy the demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |-
         python tests/fixtures.py \
           fixtures.db \
@@ -40,14 +61,18 @@ jobs:
           fixtures-metadata.json \
           plugins \
           --extra-db-filename extra_database.db
+        # Package the config with the plugins, excluding test-only plugin secrets
+        # that reference temporary files outside the deployed container.
+        jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json
     - name: Build docs.db
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
     - name: Set up the alternate-route demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         echo '
         from datasette import hookimpl
@@ -58,7 +83,8 @@ jobs:
             db.route = "alternative-route"
         ' > plugins/alternative_route.py
         cp fixtures.db fixtures2.db
-    - name: And the counters writable canned query demo
+    - name: And the counters writable stored query demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         cat > plugins/counters.py <<EOF
         from datasette import hookimpl
@@ -70,23 +96,24 @@ jobs:
                 await db.execute_write("insert or ignore into counters (name, value) values ('counter_a', 0)")
                 await db.execute_write("insert or ignore into counters (name, value) values ('counter_b', 0)")
                 await db.execute_write("insert or ignore into counters (name, value) values ('counter_c', 0)")
-            return inner
-        @hookimpl
-        def canned_queries(database):
-            if database == "counters":
-                queries = {}
                 for name in ("counter_a", "counter_b", "counter_c"):
-                    queries["increment_{}".format(name)] = {
+                    await datasette.add_query(
-                        "sql": "update counters set value = value + 1 where name = '{}'".format(name),
+                        "counters",
-                        "on_success_message_sql": "select 'Counter {name} incremented to ' || value from counters where name = '{name}'".format(name=name),
+                        "increment_{}".format(name),
-                        "write": True,
+                        "update counters set value = value + 1 where name = '{}'".format(name),
-                    }
+                        on_success_message_sql="select 'Counter {name} incremented to ' || value from counters where name = '{name}'".format(name=name),
-                    queries["decrement_{}".format(name)] = {
+                        is_write=True,
-                        "sql": "update counters set value = value - 1 where name = '{}'".format(name),
+                        is_trusted=True,
-                        "on_success_message_sql": "select 'Counter {name} decremented to ' || value from counters where name = '{name}'".format(name=name),
+                    )
-                        "write": True,
+                    await datasette.add_query(
-                    }
+                        "counters",
-                return queries
+                        "decrement_{}".format(name),
+                        "update counters set value = value - 1 where name = '{}'".format(name),
+                        on_success_message_sql="select 'Counter {name} decremented to ' || value from counters where name = '{name}'".format(name=name),
+                        is_write=True,
+                        is_trusted=True,
+                    )
+            return inner
         EOF
     # - name: Make some modifications to metadata.json
     #   run: |
@@ -97,12 +124,15 @@ jobs:
     #     cat metadata.json
     - id: auth
       name: Authenticate to Google Cloud
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/auth@v3
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}
     - name: Set up Cloud SDK
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/setup-gcloud@v3
     - name: Deploy to Cloud Run
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       env:
         LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
       run: |-
@@ -117,16 +147,16 @@ jobs:
             --plugins-dir=plugins \
             --branch=$GITHUB_SHA \
             --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \
+            --extra-options="--config plugins/fixtures-config.json --setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
             --install 'datasette-ephemeral-tables>=0.2.2' \
             --service "datasette-latest$SUFFIX" \
             --secret $LATEST_DATASETTE_SECRET
-    - name: Deploy to docs as well (only for main)
+    - name: Upload latest documentation database to S3 (only for main)
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
       run: |-
-        # Deploy docs.db to a different service
+        # Keep development documentation separate from the stable release database.
-        datasette publish cloudrun docs.db \
+        s3-credentials put-object datasette-docs latest/docs.db docs.db \
-            --branch=$GITHUB_SHA \
+          --content-type application/octet-stream
-            --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1" \
-            --service=datasette-docs-latest

.github/workflows/documentation-links.yml

@@ -1,16 +0,0 @@
-name: Read the Docs Pull Request Preview
-on:
-  pull_request_target:
-    types:
-      - opened
-
-permissions:
-  pull-requests: write
-
-jobs:
-  documentation-links:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: readthedocs/actions/preview@v1
-        with:
-          project-slug: "datasette"

.github/workflows/playwright.yml

@@ -0,0 +1,54 @@
+name: Playwright
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: [chromium, firefox, webkit]
+    steps:
+    - uses: actions/checkout@v7
+    - name: Set up Python 3.14
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.14"
+        allow-prereleases: true
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Cache uv
+      uses: actions/cache@v6
+      with:
+        path: ~/.cache/uv
+        key: ${{ runner.os }}-py3.14-uv-${{ hashFiles('pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-py3.14-uv-
+    - name: Cache Playwright browsers
+      uses: actions/cache@v6
+      with:
+        path: ~/.cache/ms-playwright/
+        key: ${{ runner.os }}-playwright-${{ matrix.browser }}-${{ hashFiles('pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-playwright-${{ matrix.browser }}-
+    - name: Install uv
+      run: python -m pip install uv
+    - name: Install dependencies
+      run: uv sync --group dev --group playwright
+    - name: Install ${{ matrix.browser }}
+      run: uv run --group dev --group playwright playwright install --with-deps ${{ matrix.browser }}
+    - name: Run Playwright tests
+      run: uv run --group dev --group playwright pytest tests/test_playwright.py --playwright --browser ${{ matrix.browser }}

.github/workflows/prettier.yml

@@ -1,6 +1,15 @@
 name: Check JavaScript for conformance with Prettier
 
-on: [push]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -10,8 +19,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out repo
-      uses: actions/checkout@v4
+      uses: actions/checkout@v7
-    - uses: actions/cache@v4
+    - uses: actions/cache@v6
       name: Configure npm caching
       with:
         path: ~/.npm

.github/workflows/publish.yml

@@ -2,7 +2,7 @@ name: Publish Python Package
 
 on:
   release:
-    types: [created]
+    types: [published]
 
 permissions:
   contents: read
@@ -14,7 +14,7 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
       with:
@@ -23,7 +23,7 @@ jobs:
         cache-dependency-path: pyproject.toml
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install . --group dev
     - name: Run tests
       run: |
         pytest
@@ -35,7 +35,7 @@ jobs:
     permissions:
       id-token: write
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -51,12 +51,14 @@ jobs:
     - name: Publish
       uses: pypa/gh-action-pypi-publish@release/v1
 
+  # After the first non-prerelease 1.0 release, disable this job on 0.65.x,
+  # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs.
   deploy_static_docs:
     runs-on: ubuntu-latest
     needs: [deploy]
     if: "!github.event.release.prerelease"
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -65,34 +67,28 @@ jobs:
         cache-dependency-path: pyproject.toml
     - name: Install dependencies
       run: |
-        python -m pip install -e .[docs]
+        python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1
+        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
     - name: Build docs.db
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
-    - id: auth
+    - name: Upload stable documentation database to S3
-      name: Authenticate to Google Cloud
+      env:
-      uses: google-github-actions/auth@v2
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
-      with:
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
-        credentials_json: ${{ secrets.GCP_SA_KEY }}
-    - name: Set up Cloud SDK
-      uses: google-github-actions/setup-gcloud@v3
-    - name: Deploy stable-docs.datasette.io to Cloud Run
       run: |-
-        gcloud config set run/region us-central1
+        s3-credentials put-object datasette-docs docs.db docs.db \
-        gcloud config set project datasette-222320
+          --content-type application/octet-stream
-        datasette publish cloudrun docs.db \
-            --service=datasette-docs-stable
 
   deploy_docker:
     runs-on: ubuntu-latest
     needs: [deploy]
     if: "!github.event.release.prerelease"
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Build and push to Docker Hub
       env:
         DOCKER_USER: ${{ secrets.DOCKER_USER }}

.github/workflows/push_docker_tag.yml

@@ -13,7 +13,7 @@ jobs:
   deploy_docker:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v7
     - name: Build and push to Docker Hub
       env:
         DOCKER_USER: ${{ secrets.DOCKER_USER }}

.github/workflows/spellcheck.yml

@@ -1,6 +1,15 @@
 name: Check spelling in documentation
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -9,7 +18,7 @@ jobs:
   spellcheck:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -18,7 +27,7 @@ jobs:
         cache-dependency-path: '**/pyproject.toml'
     - name: Install dependencies
       run: |
-        pip install -e '.[docs]'
+        pip install . --group dev
     - name: Check spelling
       run: |
         codespell README.md --ignore-words docs/codespell-ignore-words.txt

.github/workflows/stable-docs.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v5
+      uses: actions/checkout@v7
       with:
         fetch-depth: 0  # We need all commits to find docs/ changes
     - name: Set up Git user

.github/workflows/test-coverage.yml

@@ -1,40 +0,0 @@
-name: Calculate test coverage
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out datasette
-      uses: actions/checkout@v4
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version: '3.12'
-        cache: 'pip'
-        cache-dependency-path: '**/pyproject.toml'
-    - name: Install Python dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install -e .[test]
-        python -m pip install pytest-cov
-    - name: Run tests
-      run: |-
-        ls -lah
-        cat .coveragerc
-        pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x
-        ls -lah
-    - name: Upload coverage report
-      uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.xml

.github/workflows/test-pyodide.yml

@@ -2,9 +2,15 @@ name: Test in Pyodide with shot-scraper
 
 on:
   push:
+    branches:
+    - main
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 permissions:
   contents: read
 
@@ -12,7 +18,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Set up Python 3.10
       uses: actions/setup-python@v6
       with:
@@ -20,7 +26,7 @@ jobs:
         cache: 'pip'
         cache-dependency-path: '**/pyproject.toml'
     - name: Cache Playwright browsers
-      uses: actions/cache@v4
+      uses: actions/cache@v6
       with:
         path: ~/.cache/ms-playwright/
         key: ${{ runner.os }}-browsers

.github/workflows/test-sqlite-support.yml

@@ -1,6 +1,15 @@
 name: Test SQLite versions
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -12,10 +21,10 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest]
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.13"]
         sqlite-version: [
           #"3", # latest version
-          "3.46",
+          #"3.46",
           #"3.45",
           #"3.27",
           #"3.26",
@@ -25,7 +34,7 @@ jobs:
           #"3.23.1" # 2018-04-10, before UPSERT
         ]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
       with:
@@ -34,7 +43,7 @@ jobs:
         cache: pip
         cache-dependency-path: pyproject.toml
     - name: Set up SQLite ${{ matrix.sqlite-version }}
-      uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6
+      uses: ./.github/actions/setup-sqlite-version
       with:
         version: ${{ matrix.sqlite-version }}
         cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1"
@@ -45,7 +54,7 @@ jobs:
         (cd tests && gcc ext.c -fPIC -shared -o ext.so)
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install . --group dev
         pip freeze
     - name: Run tests
       run: |

.github/workflows/test.yml

@@ -1,6 +1,15 @@
 name: Test
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -9,35 +18,56 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
+        include:
+        - python-version: "3.14"
+          coverage: true
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v6
+      uses: actions/setup-python@v7
       with:
         python-version: ${{ matrix.python-version }}
         allow-prereleases: true
         cache: pip
         cache-dependency-path: pyproject.toml
+        check-latest: true
     - name: Build extension for --load-extension test
       run: |-
         (cd tests && gcc ext.c -fPIC -shared -o ext.so)
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install . --group dev
         pip freeze
+    - name: Install pytest-cov
+      if: ${{ matrix.coverage }}
+      run: pip install pytest-cov
     - name: Run tests
       run: |
+        if [ "${{ matrix.coverage }}" = "true" ]; then
+          COV="--cov=datasette --cov-config=.coveragerc"
+          pytest -n auto -m "not serial" $COV --cov-report=
+          pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term
+        else
           pytest -n auto -m "not serial"
           pytest -m "serial"
+        fi
         # And the test that exceeds a localhost HTTPS server
         tests/test_datasette_https_server.sh
-    - name: Install docs dependencies
+    - name: Upload coverage report
-      run: |
+      if: ${{ matrix.coverage }}
-        pip install -e '.[docs]'
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: coverage.xml
     - name: Black
-      run: black --check .
+      run: |
+        black --version
+        black --check .
+    - name: Ruff
+      run: ruff check datasette tests
     - name: Check if cog needs to be run
       run: |
         cog --check docs/*.rst

.github/workflows/tmate-mac.yml

@@ -10,6 +10,6 @@ jobs:
   build:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v7
     - name: Setup tmate session
       uses: mxschmitt/action-tmate@v3

.github/workflows/tmate.yml

@@ -11,7 +11,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v7
     - name: Setup tmate session
       uses: mxschmitt/action-tmate@v3
       env:

.gitignore

@@ -1,13 +1,20 @@
 build-metadata.json
 datasets.json
 
+.playwright-mcp
+
 scratchpad
 
+ignored/
+
 .vscode
 
 uv.lock
 data.db
 
+# test databases
+*.db
+
 # We don't use Pipfile, so ignore them
 Pipfile
 Pipfile.lock
@@ -127,3 +134,5 @@ node_modules
 tests/*.dylib
 tests/*.so
 tests/*.dll
+
+.idea

.readthedocs.yaml

@@ -1,16 +1,17 @@
 version: 2
 
-build:
-  os: ubuntu-20.04
-  tools:
-    python: "3.11"
-
 sphinx:
   configuration: docs/conf.py
 
-python:
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+  jobs:
     install:
-  - method: pip
+    - pip install --upgrade pip
-    path: .
+    - pip install . --group dev
-    extra_requirements:
+
-    - docs
+formats:
+- pdf
+- epub

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.0-slim-bullseye as build
+FROM python:3.11-slim-bookworm AS build
 
 # Version of Datasette to install, e.g. 0.55
 #   docker build . -t datasette --build-arg VERSION=0.55

Justfile

@@ -5,39 +5,65 @@ export DATASETTE_SECRET := "not_a_secret"
 
 # Setup project
 @init:
-  uv sync --extra test --extra docs
+  uv sync
 
 # Run pytest with supplied options
 @test *options: init
   uv run pytest -n auto {{options}}
 
+# Install Playwright browser support, Chromium by default
+@playwright-install browser="chromium":
+  uv run --group playwright playwright install {{browser}}
+
+# Install all Playwright browsers used by the test suite
+@playwright-install-all:
+  uv run --group playwright playwright install chromium firefox webkit
+
+# Run Playwright tests, Chromium by default
+@playwright browser="chromium" *options:
+  uv run --group playwright pytest tests/test_playwright.py --playwright --browser {{browser}} {{options}}
+
+# Run Playwright tests against all supported browsers
+@playwright-all *options:
+  uv run --group playwright pytest tests/test_playwright.py --playwright --browser chromium --browser firefox --browser webkit {{options}}
+
 @codespell:
   uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt
   uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt
   uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt
   uv run codespell tests --ignore-words docs/codespell-ignore-words.txt
 
-# Run linters: black, flake8, mypy, cog
+# Run linters: black, ruff, prettier, cog
 @lint: codespell
-  uv run black . --check
+  uv run black datasette tests --check
-  uv run flake8
+  uv run ruff check datasette tests
-  uv run --extra test cog --check README.md docs/*.rst
+  npm run prettier -- --check
+  uv run cog --check README.md docs/*.rst
+
+# Apply ruff fixes
+@fix:
+  uv run ruff check --fix datasette tests
 
 # Rebuild docs with cog
 @cog:
-  uv run --extra test cog -r README.md docs/*.rst
+  uv run cog -r README.md docs/*.rst
 
 # Serve live docs on localhost:8000
-@docs: cog blacken-docs
+@docs: shots cog blacken-docs
-  uv run --extra docs make -C docs livehtml
+  uv run make -C docs livehtml
 
 # Build docs as static HTML
 @docs-build: cog blacken-docs
   rm -rf docs/_build && cd docs && uv run make html
 
+# Take any missing documentation screenshots defined in docs/shots.yml
+@shots:
+  uv run --group shots shot-scraper install
+  cd docs && uv run --group shots shot-scraper multi shots.yml --no-clobber --reduced-motion --retina
+
 # Apply Black
 @black:
-  uv run black .
+  uv run black datasette tests
 
 # Apply blacken-docs
 @blacken-docs:

README.md

@@ -36,7 +36,7 @@ You can also install it using `pip` or `pipx`:
 
     pip install datasette
 
-Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
+Datasette requires Python 3.10 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
 
 ## Basic usage
 

datasette/__init__.py

@@ -1,7 +1,15 @@
 from datasette.permissions import Permission  # noqa
 from datasette.version import __version_info__, __version__  # noqa
 from datasette.events import Event  # noqa
-from datasette.utils.asgi import Forbidden, NotFound, Request, Response  # noqa
+from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor  # noqa
+from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions  # noqa
+from datasette.utils.asgi import (  # noqa
+    Forbidden,
+    NotFound,
+    PayloadTooLarge,
+    Request,
+    Response,
+)
 from datasette.utils import actor_matches_allow  # noqa
 from datasette.views import Context  # noqa
 from .hookspecs import hookimpl  # noqa

datasette/_pytest_plugin.py

@@ -0,0 +1,124 @@
+"""
+Pytest plugin that automatically closes any Datasette instances constructed
+during a pytest test — both in the test body and in function-scoped
+fixtures. Instances constructed by session-, module-, class- or package-
+scoped fixtures are left alone, because other tests in the session will
+still want to use them.
+
+Registered as a pytest11 entry point in pyproject.toml so that downstream
+projects using Datasette get the same FD-safety net for their own tests.
+
+Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the
+equivalent ini file).
+"""
+
+from __future__ import annotations
+
+import contextvars
+import weakref
+
+import pytest
+
+_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar(
+    "datasette_active_instances", default=None
+)
+
+_original_init = None
+
+
+def _install_tracking():
+    # datasette.app is imported lazily here rather than at module level:
+    # as a pytest11 entry point this module is imported during pytest
+    # startup, before pytest-cov starts measuring, so a module-level
+    # import would drag in all of datasette and make every import-time
+    # line in the package invisible to coverage
+    global _original_init
+    if _original_init is not None:
+        return
+    from datasette.app import Datasette
+
+    _original_init = Datasette.__init__
+
+    def _tracking_init(self, *args, **kwargs):
+        _original_init(self, *args, **kwargs)
+        instances = _active_instances.get()
+        if instances is not None:
+            instances.append(weakref.ref(self))
+
+    Datasette.__init__ = _tracking_init
+
+
+def pytest_configure(config):
+    if _enabled(config):
+        _install_tracking()
+
+
+def pytest_addoption(parser):
+    parser.addini(
+        "datasette_autoclose",
+        help=(
+            "Automatically close Datasette instances created inside test "
+            "bodies and function-scoped fixtures (default: true)."
+        ),
+        default="true",
+    )
+
+
+def _enabled(config) -> bool:
+    value = config.getini("datasette_autoclose")
+    if isinstance(value, bool):
+        return value
+    return str(value).strip().lower() not in ("false", "0", "no", "off")
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_protocol(item, nextitem):
+    """Track Datasette instances across setup, call and teardown; close at end."""
+    if not _enabled(item.config):
+        yield
+        return
+    refs: list[weakref.ref] = []
+    token = _active_instances.set(refs)
+    try:
+        yield
+    finally:
+        _active_instances.reset(token)
+        for ref in reversed(refs):
+            ds = ref()
+            if ds is None:
+                continue
+            try:
+                ds.close()
+            except Exception as e:  # noqa: BLE001
+                # Surfaced as a pytest warning; teardown must not fail the run
+                item.warn(
+                    pytest.PytestUnraisableExceptionWarning(
+                        f"Error closing Datasette instance: {e!r}"
+                    )
+                )
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_fixture_setup(fixturedef, request):
+    """Exempt instances created by non-function-scoped fixtures.
+
+    Session-, module-, class- and package-scoped fixtures produce Datasette
+    instances that must survive beyond the current test — other tests in
+    the session will still use them. When such a fixture creates one or
+    more Datasette instances during its setup, we snapshot the tracking
+    list before the fixture runs and subtract off any instances that were
+    added during its setup, so they don't get closed at test teardown.
+    """
+    refs = _active_instances.get()
+    if refs is None:
+        yield
+        return
+    before_ids = {id(ref) for ref in refs}
+    yield
+    if fixturedef.scope != "function":
+        new_refs = [ref for ref in refs if id(ref) not in before_ids]
+        for new_ref in new_refs:
+            try:
+                refs.remove(new_ref)
+            except ValueError:
+                pass

datasette/actor_auth_cookie.py

@@ -1,8 +1,10 @@
-from datasette import hookimpl
-from itsdangerous import BadSignature
-from datasette.utils import baseconv
 import time
 
+from itsdangerous import BadSignature
+
+from datasette import hookimpl
+from datasette.utils import baseconv
+
 
 @hookimpl
 def actor_from_request(datasette, request):

datasette/background_tasks.py

@@ -0,0 +1,227 @@
+"""
+Supervised background-task registration for Datasette core.
+
+Plugins that need long-lived background work (a polling loop, a queue
+consumer, a scheduled job runner) register it with
+``datasette.add_background_task(func, name=None)`` - typically from a
+``startup`` plugin hook - instead of fire-and-forgetting their own
+``asyncio.create_task()``. Core owns:
+
+- **references**: every launched ``asyncio.Task`` is kept alive on a
+  :class:`BackgroundTaskSupervisor`, so it can never be silently garbage
+  collected the way an unreferenced ``create_task()`` call can be;
+- **launch timing**: registered work is buffered until
+  :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to
+  happen only after *every* plugin's ``startup`` hook has finished - so
+  a task that depends on another plugin having registered something first
+  doesn't need ``tryfirst=True`` ordering tricks;
+- **crash surfacing**: an unhandled exception in a background task is
+  logged with its full traceback to the ``datasette.background_tasks``
+  logger and recorded on the handle, instead of becoming an "Task
+  exception was never retrieved" warning nobody sees;
+- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels
+  every task still running and waits (with a grace period) for them to
+  actually stop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import datetime
+import functools
+import logging
+from collections.abc import Awaitable, Callable
+
+logger = logging.getLogger("datasette.background_tasks")
+
+
+def _utcnow_iso() -> str:
+    return datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+
+def _function_path(func: Callable) -> str:
+    """Describe the callable without guessing which plugin registered it."""
+    while isinstance(func, functools.partial):
+        func = func.func
+    if not hasattr(func, "__qualname__"):
+        func = type(func).__call__
+    return f"{func.__module__}.{func.__qualname__}"
+
+
+class BackgroundTask:
+    """A handle to a single piece of supervised background work.
+
+    States: ``registered`` (added but not yet launched) -> ``running`` ->
+    one of ``completed`` (returned cleanly), ``crashed`` (raised an
+    exception other than ``CancelledError`` - see ``.exception``), or
+    ``cancelled`` (``.cancel()`` was called, or it was still running at
+    shutdown).
+    """
+
+    def __init__(
+        self,
+        name: str,
+        func: Callable[[object], Awaitable[None]],
+    ):
+        self.name = name
+        self.state = "registered"
+        self.task: asyncio.Task | None = None
+        self.exception: BaseException | None = None
+        self.started_at: str | None = None
+        self.function = _function_path(func)
+        self._func = func
+        self._supervisor: BackgroundTaskSupervisor | None = None
+
+    def cancel(self) -> None:
+        """Cancel this task.
+
+        If it has already been launched, cancels the underlying
+        ``asyncio.Task`` - its state becomes ``cancelled`` once the
+        cancellation is observed (asynchronously, via the task's done
+        callback). If it has not been launched yet, this is a no-op as
+        far as asyncio is concerned (there's no task to cancel) but it
+        deregisters the handle from its supervisor so it never runs.
+        """
+        if self.task is not None:
+            self.task.cancel()
+        elif self._supervisor is not None:
+            self._supervisor._deregister(self)
+
+    def __repr__(self) -> str:
+        return f"<BackgroundTask name={self.name!r} state={self.state!r}>"
+
+
+class BackgroundTaskSupervisor:
+    """Owns registration and launch of every :class:`BackgroundTask` for a
+    single ``Datasette`` instance.
+
+    Registration (:meth:`add`) is separate from launch
+    (:meth:`launch_all`): plugins register work whenever convenient
+    (typically from a ``startup`` hook, but request handlers can register
+    dynamic per-job work too), and it either sits buffered until
+    :meth:`launch_all` runs, or - if :meth:`launch_all` has already run -
+    starts immediately.
+
+    Strong references to every :class:`BackgroundTask` (and its
+    ``asyncio.Task``) are kept for the life of the instance, by design -
+    that's what makes the enrichments-style "fire-and-forget task gets
+    garbage collected mid-flight" bug impossible here. There is currently
+    no pruning of completed/crashed/cancelled tasks, so a plugin that
+    dynamically registers many short-lived tasks over a long process
+    lifetime (a per-job registration pattern, e.g. one task per queued
+    job) will grow this list without bound. That's an accepted v1
+    trade-off in favour of full introspection (``/-/tasks``); revisit
+    with a pruning or capping policy if unbounded growth is reported in
+    practice.
+    """
+
+    def __init__(self, datasette):
+        self._datasette = datasette
+        self._tasks: list[BackgroundTask] = []
+        self._names = set()
+        self._launched = False
+        self._lock = asyncio.Lock()
+
+    def add(self, func, name=None) -> BackgroundTask:
+        base_name = name or getattr(func, "__qualname__", None) or repr(func)
+        actual_name = self._unique_name(base_name)
+        handle = BackgroundTask(actual_name, func)
+        handle._supervisor = self
+        self._tasks.append(handle)
+        self._names.add(actual_name)
+        if self._launched:
+            self._launch_one(handle)
+        return handle
+
+    def _unique_name(self, base_name: str) -> str:
+        if base_name not in self._names:
+            return base_name
+        n = 2
+        while f"{base_name}-{n}" in self._names:
+            n += 1
+        return f"{base_name}-{n}"
+
+    def _deregister(self, handle: BackgroundTask) -> None:
+        try:
+            self._tasks.remove(handle)
+        except ValueError:
+            pass
+        self._names.discard(handle.name)
+
+    def _launch_one(self, handle: BackgroundTask) -> None:
+        handle.state = "running"
+        handle.started_at = _utcnow_iso()
+        handle.task = asyncio.create_task(
+            handle._func(self._datasette), name=handle.name
+        )
+        handle.task.add_done_callback(functools.partial(_on_task_done, handle))
+
+    async def launch_all(self) -> None:
+        """Launch every currently-registered task that hasn't launched
+        yet. Idempotent and safe to call concurrently: subsequent (or
+        racing) calls are no-ops once the first has set ``self._launched``.
+        """
+        if self._launched:
+            return
+        async with self._lock:
+            if self._launched:
+                return
+            self._launched = True
+            for handle in list(self._tasks):
+                if handle.task is None:
+                    self._launch_one(handle)
+
+    async def cancel_all(self, grace: float = 5.0) -> None:
+        """Cancel every task that isn't already done, then wait up to
+        ``grace`` seconds for them to actually finish. Stragglers still
+        running after that are logged by name (but left to finish or not
+        on their own - this does not forcibly kill them, asyncio has no
+        mechanism for that).
+        """
+        handles_by_task = {
+            handle.task: handle for handle in self._tasks if handle.task is not None
+        }
+        pending = [task for task in handles_by_task if not task.done()]
+        for task in pending:
+            task.cancel()
+        if not pending:
+            return
+        _done, not_done = await asyncio.wait(pending, timeout=grace)
+        if not_done:
+            names = sorted(handles_by_task[task].name for task in not_done)
+            logger.warning(
+                "%d background task(s) did not finish within the %.1fs grace "
+                "period after cancellation: %s",
+                len(names),
+                grace,
+                ", ".join(names),
+            )
+
+    def tasks(self) -> list[BackgroundTask]:
+        """Return every registered :class:`BackgroundTask`, launched or
+        not, in registration order. Used by the ``/-/tasks`` debug
+        endpoint.
+        """
+        return list(self._tasks)
+
+    @property
+    def launched(self) -> bool:
+        """Whether :meth:`launch_all` has run yet - lets ``/-/tasks``
+        distinguish "no tasks registered" from "tasks registered but
+        nothing has armed the launch yet" without reaching for the
+        private ``_launched`` attribute.
+        """
+        return self._launched
+
+
+def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None:
+    if task.cancelled():
+        handle.state = "cancelled"
+        return
+    exc = task.exception()
+    if exc is not None:
+        handle.state = "crashed"
+        handle.exception = exc
+        logger.error("Background task %r crashed", handle.name, exc_info=exc)
+        return
+    handle.state = "completed"

datasette/blob_renderer.py

@@ -1,8 +1,9 @@
-from datasette import hookimpl
-from datasette.utils.asgi import Response, BadRequest
-from datasette.utils import to_css_class
 import hashlib
 
+from datasette import hookimpl
+from datasette.utils import to_css_class
+from datasette.utils.asgi import BadRequest, Response
+
 _BLOB_COLUMN = "_blob_column"
 _BLOB_HASH = "_blob_hash"
 

datasette/cli.py

@@ -1,42 +1,45 @@
 import asyncio
-import uvicorn
-import click
-from click import formatting
-from click.types import CompositeParamType
-from click_default_group import DefaultGroup
 import functools
 import json
 import os
 import pathlib
-from runpy import run_module
 import shutil
-from subprocess import call
 import sys
 import textwrap
 import webbrowser
+from runpy import run_module
+from subprocess import call
+
+import click
+import uvicorn
+from click import formatting
+from click.types import CompositeParamType
+from click_default_group import DefaultGroup
+
 from .app import (
-    Datasette,
     DEFAULT_SETTINGS,
     SETTINGS,
     SQLITE_LIMIT_ATTACHED,
+    Datasette,
     pm,
 )
+from .inspect import inspect_tables
 from .utils import (
+    ConnectionProblem,
     LoadExtension,
+    SpatialiteConnectionProblem,
+    SpatialiteNotFound,
     StartupError,
+    StaticMount,
+    ValueAsBooleanError,
     check_connection,
     deep_dict_update,
     find_spatialite,
-    parse_metadata,
-    ConnectionProblem,
-    SpatialiteConnectionProblem,
     initial_path_for_datasette,
     pairs_to_nested_config,
+    parse_metadata,
     temporary_docker_directory,
     value_as_boolean,
-    SpatialiteNotFound,
-    StaticMount,
-    ValueAsBooleanError,
 )
 from .utils.sqlite import sqlite3
 from .utils.testing import TestClient
@@ -74,7 +77,7 @@ class Setting(CompositeParamType):
             # Datasette 1.0, we turn bare setting names into setting.name
             # Type checking for those older settings
             default = DEFAULT_SETTINGS[name]
-            name = "settings.{}".format(name)
+            name = f"settings.{name}"
             if isinstance(default, bool):
                 try:
                     return name, "true" if value_as_boolean(value) else "false"
@@ -109,15 +112,11 @@ def sqlite_extensions(fn):
             return fn(*args, **kwargs)
         except AttributeError as e:
             if "enable_load_extension" in str(e):
-                raise click.ClickException(
+                raise click.ClickException(textwrap.dedent("""
-                    textwrap.dedent(
-                        """
                     Your Python installation does not have the ability to load SQLite extensions.
 
                     More information: https://datasette.io/help/extensions
-                    """
+                    """).strip())
-                    ).strip()
-                )
             raise
 
     return wrapped
@@ -158,14 +157,18 @@ async def inspect_(files, sqlite_extensions):
     app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
     data = {}
     for name, database in app.databases.items():
-        counts = await database.table_counts(limit=3600 * 1000)
+
+        def _inspect_tables(conn):
+            return inspect_tables(conn, {})
+
+        tables = await database.execute_fn(_inspect_tables)
         data[name] = {
             "hash": database.hash,
             "size": database.size,
             "file": database.path,
             "tables": {
-                table_name: {"count": table_count}
+                table_name: {"count": table["count"]}
-                for table_name, table_count in counts.items()
+                for table_name, table in tables.items()
             },
         }
     return data
@@ -174,7 +177,6 @@ async def inspect_(files, sqlite_extensions):
 @cli.group()
 def publish():
     """Publish specified SQLite database files to the internet along with a Datasette-powered interface and API"""
-    pass
 
 
 # Register publish plugins
@@ -499,6 +501,7 @@ def uninstall(packages, yes):
     "--internal",
     type=click.Path(),
     help="Path to a persistent Datasette internal SQLite database",
+    envvar="DATASETTE_INTERNAL",
 )
 def serve(
     files,
@@ -551,7 +554,7 @@ def serve(
     if reload:
         import hupper
 
-        reloader = hupper.start_reloader("datasette.cli.serve")
+        reloader = hupper.start_reloader("datasette.cli.cli")
         if immutable:
             reloader.watch_files(immutable)
         if config:
@@ -581,27 +584,27 @@ def serve(
         # https://github.com/simonw/datasette/issues/2389
         deep_dict_update(config_data, settings_updates)
 
-    kwargs = dict(
+    kwargs = {
-        immutables=immutable,
+        "immutables": immutable,
-        cache_headers=not reload,
+        "cache_headers": not reload,
-        cors=cors,
+        "cors": cors,
-        inspect_data=inspect_data,
+        "inspect_data": inspect_data,
-        config=config_data,
+        "config": config_data,
-        metadata=metadata_data,
+        "metadata": metadata_data,
-        sqlite_extensions=sqlite_extensions,
+        "sqlite_extensions": sqlite_extensions,
-        template_dir=template_dir,
+        "template_dir": template_dir,
-        plugins_dir=plugins_dir,
+        "plugins_dir": plugins_dir,
-        static_mounts=static,
+        "static_mounts": static,
-        settings=None,  # These are passed in config= now
+        "settings": None,  # These are passed in config= now
-        memory=memory,
+        "memory": memory,
-        secret=secret,
+        "secret": secret,
-        version_note=version_note,
+        "version_note": version_note,
-        pdb=pdb,
+        "pdb": pdb,
-        crossdb=crossdb,
+        "crossdb": crossdb,
-        nolock=nolock,
+        "nolock": nolock,
-        internal=internal,
+        "internal": internal,
-        default_deny=default_deny,
+        "default_deny": default_deny,
-    )
+    }
 
     # Separate directories from files
     directories = [f for f in files if os.path.isdir(f)]
@@ -619,12 +622,12 @@ def serve(
     for file in file_paths:
         if not pathlib.Path(file).exists():
             if create:
-                sqlite3.connect(file).execute("vacuum")
+                conn = sqlite3.connect(file)
+                conn.execute("vacuum")
+                conn.close()
             else:
                 raise click.ClickException(
-                    "Invalid value for '[FILES]...': Path '{}' does not exist.".format(
+                    f"Invalid value for '[FILES]...': Path '{file}' does not exist."
-                        file
-                    )
                 )
 
     # Check for duplicate files by resolving all paths to their absolute forms
@@ -665,12 +668,6 @@ def serve(
         # Private utility mechanism for writing unit tests
         return ds
 
-    # Run the "startup" plugin hooks
-    run_sync(ds.invoke_startup)
-
-    # Run async soundness checks - but only if we're not under pytest
-    run_sync(lambda: check_databases(ds))
-
     if headers and not get:
         raise click.ClickException("--headers can only be used with --get")
 
@@ -678,10 +675,23 @@ def serve(
         raise click.ClickException("--token can only be used with --get")
 
     if get:
+        # --get means we don't run Uvicorn at all
+        run_sync(lambda: check_databases(ds))
+
+        try:
+            run_sync(ds.invoke_startup)
+        except StartupError as e:
+            raise click.ClickException(e.args[0])
+
+        # --get never launches background tasks: TestClient's request below
+        # flows through the full ASGI stack, including the
+        # AsgiRunOnFirstRequest fallback, which would otherwise launch them.
+        ds._suppress_background_tasks = True
+
         client = TestClient(ds)
         request_headers = {}
         if token:
-            request_headers["Authorization"] = "Bearer {}".format(token)
+            request_headers["Authorization"] = f"Bearer {token}"
         cookies = {}
         if actor:
             cookies["ds_actor"] = client.actor_cookie(json.loads(actor))
@@ -702,6 +712,23 @@ def serve(
         sys.exit(exit_code)
         return
 
+    # check_databases, invoke_startup() and the uvicorn server all run on a
+    # single event loop, so that anything a plugin's "startup" hook schedules
+    # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is
+    # still alive when the server starts handling requests.
+    async def _serve_async():
+        # Populate internal catalog tables before invoke_startup
+        await check_databases(ds)
+
+        # Run the full startup sequence (immutable-database table-count
+        # precompute + the "startup" plugin hooks) via the same entry point
+        # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when
+        # uvicorn's lifespan.startup fires moments later.
+        try:
+            await ds._startup_sequence()
+        except StartupError as e:
+            raise click.ClickException(e.args[0])
+
         # Start the server
         url = None
         if root:
@@ -713,19 +740,26 @@ def serve(
         if open_browser:
             if url is None:
                 # Figure out most convenient URL - to table, database or homepage
-            path = run_sync(lambda: initial_path_for_datasette(ds))
+                path = await initial_path_for_datasette(ds)
                 url = f"http://{host}:{port}{path}"
             webbrowser.open(url)
-    uvicorn_kwargs = dict(
+        uvicorn_kwargs = {
-        host=host, port=port, log_level="info", lifespan="on", workers=1
+            "host": host,
-    )
+            "port": port,
+            "log_level": "info",
+            "lifespan": "on",
+            "workers": 1,
+        }
         if uds:
             uvicorn_kwargs["uds"] = uds
         if ssl_keyfile:
             uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile
         if ssl_certfile:
             uvicorn_kwargs["ssl_certfile"] = ssl_certfile
-    uvicorn.run(ds.app(), **uvicorn_kwargs)
+        server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs))
+        await server.serve()
+
+    asyncio.run(_serve_async())
 
 
 @cli.command()
@@ -815,7 +849,10 @@ def create_token(
     ds = Datasette(secret=secret, plugins_dir=plugins_dir)
 
     # Run ds.invoke_startup() in an event loop
+    try:
         run_sync(ds.invoke_startup)
+    except StartupError as e:
+        raise click.ClickException(e.args[0])
 
     # Warn about any unknown actions
     actions = []
@@ -830,21 +867,23 @@ def create_token(
                 err=True,
             )
 
-    restrict_database = {}
+    from datasette.tokens import TokenRestrictions
-    for database, action in databases:
-        restrict_database.setdefault(database, []).append(action)
-    restrict_resource = {}
-    for database, resource, action in resources:
-        restrict_resource.setdefault(database, {}).setdefault(resource, []).append(
-            action
-        )
 
-    token = ds.create_token(
+    restrictions = TokenRestrictions()
+    for action in alls:
+        restrictions.allow_all(action)
+    for database, action in databases:
+        restrictions.allow_database(database, action)
+    for database, resource, action in resources:
+        restrictions.allow_resource(database, resource, action)
+
+    token = run_sync(
+        lambda: ds.create_token(
             id,
             expires_after=expires_after,
-        restrict_all=alls,
+            restrictions=restrictions,
-        restrict_database=restrict_database,
+            handler="signed",
-        restrict_resource=restrict_resource,
+        )
     )
     click.echo(token)
     if debug:
@@ -877,7 +916,7 @@ async def check_databases(ds):
             )
         except ConnectionProblem as e:
             raise click.UsageError(
-                f"Connection to {database.path} failed check: {str(e.args[0])}"
+                f"Connection to {database.path} failed check: {e.args[0]!s}"
             )
     # If --crossdb and more than SQLITE_LIMIT_ATTACHED show warning
     if (
@@ -885,9 +924,5 @@ async def check_databases(ds):
         and len([db for db in ds.databases.values() if not db.is_memory])
         > SQLITE_LIMIT_ATTACHED
     ):
-        msg = (
+        msg = f"Warning: --crossdb only works with the first {SQLITE_LIMIT_ATTACHED} attached databases"
-            "Warning: --crossdb only works with the first {} attached databases".format(
-                SQLITE_LIMIT_ATTACHED
-            )
-        )
         click.echo(click.style(msg, bold=True, fg="yellow"), err=True)

datasette/column_types.py

@@ -0,0 +1,81 @@
+from enum import Enum
+
+
+class SQLiteType(Enum):
+    TEXT = "TEXT"
+    INTEGER = "INTEGER"
+    REAL = "REAL"
+    BLOB = "BLOB"
+    NUMERIC = "NUMERIC"
+
+    @classmethod
+    def from_declared_type(cls, declared_type: str | None) -> "SQLiteType":
+        if declared_type is None:
+            return cls.BLOB
+
+        normalized = declared_type.strip().upper()
+        if not normalized:
+            return cls.BLOB
+
+        if "INT" in normalized:
+            return cls.INTEGER
+        if any(token in normalized for token in ("CHAR", "CLOB", "TEXT")):
+            return cls.TEXT
+        if "BLOB" in normalized:
+            return cls.BLOB
+        if any(
+            token in normalized
+            for token in ("REAL", "FLOA", "DOUB")  # codespell:ignore doub
+        ):
+            return cls.REAL
+
+        return cls.NUMERIC
+
+
+class ColumnType:
+    """
+    Base class for column types.
+
+    Subclasses must define ``name`` and ``description`` as class attributes:
+
+    - ``name``: Unique identifier string. Lowercase, no spaces.
+      Examples: "markdown", "file", "email", "url", "point", "image".
+    - ``description``: Human-readable label for admin UI dropdowns.
+      Examples: "Markdown text", "File reference", "Email address".
+    - ``sqlite_types``: Optional tuple of SQLiteType values restricting
+      which SQLite column types this ColumnType can be assigned to.
+
+    Instantiate with an optional ``config`` dict to bind per-column
+    configuration::
+
+        ct = MyColumnType(config={"key": "value"})
+        ct.config  # {"key": "value"}
+    """
+
+    name: str
+    description: str
+    sqlite_types: tuple[SQLiteType, ...] | None = None
+
+    def __init__(self, config=None):
+        self.config = config
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        """
+        Return an HTML string to render this cell value, or None to
+        fall through to the default render_cell plugin hook chain.
+        """
+        return
+
+    async def validate(self, value, datasette):
+        """
+        Validate a value before it is written. Return None if valid,
+        or a string error message if invalid.
+        """
+        return
+
+    async def transform_value(self, value, datasette):
+        """
+        Transform a value before it appears in JSON API output.
+        Return the transformed value. Default: return unchanged.
+        """
+        return value

datasette/csrf.py

@@ -0,0 +1,177 @@
+"""
+Header-based CSRF (Cross-Origin) protection.
+
+Datasette uses the Sec-Fetch-Site + Origin header approach described in
+Filippo Valsorda's article (https://words.filippo.io/csrf/) and implemented
+in Go 1.25's http.CrossOriginProtection. This replaces the previous
+token-based asgi-csrf mechanism.
+"""
+
+from __future__ import annotations
+
+import secrets
+import urllib.parse
+
+from .utils.asgi import asgi_send
+
+SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
+
+DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443}
+
+
+def _normalize_headers(raw_headers):
+    """Lowercase header names; for duplicates, last value wins."""
+    result = {}
+    for name, value in raw_headers:
+        if isinstance(name, str):
+            name = name.encode("latin-1")
+        if isinstance(value, str):
+            value = value.encode("latin-1")
+        result[name.lower()] = value
+    return result
+
+
+def _origin_tuple(value):
+    """
+    Parse an origin-like string into ``(scheme, host, port)`` with default
+    ports filled in. Raises ``ValueError`` for malformed input.
+    """
+    parsed = urllib.parse.urlsplit(value)
+    scheme = (parsed.scheme or "").lower()
+    host = (parsed.hostname or "").lower()
+    if not scheme or not host:
+        raise ValueError(f"missing scheme or host in {value!r}")
+    port = parsed.port  # may raise ValueError on bad ports
+    if port is None:
+        port = DEFAULT_PORTS.get(scheme)
+    if port is None:
+        raise ValueError(f"unknown default port for scheme {scheme!r}")
+    return scheme, host, port
+
+
+def _install_legacy_csrftoken(scope):
+    """
+    Populate ``scope["csrftoken"]`` with a callable returning a per-request
+    random token. Provided for plugin compatibility only - core no longer
+    uses this value for CSRF enforcement.
+    """
+
+    def csrftoken():
+        if "_datasette_legacy_csrftoken" not in scope:
+            scope["_datasette_legacy_csrftoken"] = secrets.token_urlsafe(32)
+        return scope["_datasette_legacy_csrftoken"]
+
+    scope["csrftoken"] = csrftoken
+
+
+class CrossOriginProtectionMiddleware:
+    """
+    Modern CSRF protection using the Sec-Fetch-Site and Origin headers.
+
+    Based on Filippo Valsorda's algorithm, as implemented in Go 1.25's
+    http.CrossOriginProtection. See https://words.filippo.io/csrf/
+
+    Unsafe-method requests are allowed through only if they look same-origin.
+    Non-browser clients (curl, etc.) send neither Sec-Fetch-Site nor Origin
+    and are passed through unchanged - CSRF is a browser-only attack.
+    """
+
+    SAFE_METHODS = SAFE_METHODS
+
+    def __init__(self, app, datasette):
+        self.app = app
+        self.datasette = datasette
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        _install_legacy_csrftoken(scope)
+
+        if scope.get("method", "GET") in self.SAFE_METHODS:
+            await self.app(scope, receive, send)
+            return
+
+        headers = _normalize_headers(scope.get("headers") or [])
+
+        authorization = headers.get(b"authorization", b"").decode("latin-1")
+        cookie_header = headers.get(b"cookie")
+        # Bearer-token requests are not ambient browser credentials, so they
+        # are not CSRF-vulnerable. Narrowly exempt them from the header check
+        # before evaluating Sec-Fetch-Site / Origin. Only "Bearer" is exempt;
+        # schemes like Basic or Digest can be browser-managed and ambient.
+        # If the request also carries a Cookie header, ambient cookie auth
+        # could be in play, so do NOT treat it as exempt.
+        if authorization and not cookie_header:
+            parts = authorization.split(None, 1)
+            if parts and parts[0].lower() == "bearer":
+                await self.app(scope, receive, send)
+                return
+
+        origin_bytes = headers.get(b"origin")
+        sec_fetch_site_bytes = headers.get(b"sec-fetch-site")
+        host_bytes = headers.get(b"host", b"")
+        origin = origin_bytes.decode("latin-1") if origin_bytes else None
+        sec_fetch_site = (
+            sec_fetch_site_bytes.decode("latin-1") if sec_fetch_site_bytes else None
+        )
+        host = host_bytes.decode("latin-1")
+
+        # Primary defense: Sec-Fetch-Site (set by browsers, unforgeable from JS)
+        if sec_fetch_site is not None:
+            if sec_fetch_site in ("same-origin", "none"):
+                await self.app(scope, receive, send)
+                return
+            await self._forbid(
+                send,
+                f"Sec-Fetch-Site was {sec_fetch_site!r}, expected 'same-origin' or 'none'",
+            )
+            return
+
+        # No Sec-Fetch-Site and no Origin -> non-browser client (curl, API, etc.)
+        if origin is None:
+            await self.app(scope, receive, send)
+            return
+
+        # Fallback for older browsers: Origin must match the request's own
+        # scheme + host + port. Compare full origin tuples, not host alone.
+        request_scheme = self._request_scheme(scope)
+        try:
+            origin_tuple = _origin_tuple(origin)
+            expected_tuple = _origin_tuple(f"{request_scheme}://{host}")
+        except ValueError:
+            await self._forbid(
+                send,
+                f"Malformed Origin {origin!r} or Host {host!r}",
+            )
+            return
+
+        if origin_tuple == expected_tuple:
+            await self.app(scope, receive, send)
+            return
+
+        await self._forbid(
+            send,
+            f"Origin {origin!r} does not match Host {host!r}",
+        )
+
+    def _request_scheme(self, scope):
+        if self.datasette is not None:
+            try:
+                if self.datasette.setting("force_https_urls"):
+                    return "https"
+            except Exception:  # noqa: BLE001, S110
+                # Settings may not be readable this early; fall back to the ASGI scheme
+                pass
+        return scope.get("scheme") or "http"
+
+    async def _forbid(self, send, reason):
+        await asgi_send(
+            send,
+            content=await self.datasette.render_template(
+                "csrf_error.html", {"reason": reason}
+            ),
+            status=403,
+            content_type="text/html; charset=utf-8",
+        )

datasette/default_actions.py

@@ -2,8 +2,8 @@ from datasette import hookimpl
 from datasette.permissions import Action
 from datasette.resources import (
     DatabaseResource,
-    TableResource,
     QueryResource,
+    TableResource,
 )
 
 
@@ -48,12 +48,32 @@ def register_actions():
             resource_class=DatabaseResource,
             also_requires="view-database",
         ),
+        Action(
+            name="execute-write-sql",
+            abbr="ews",
+            description="Execute writable SQL queries",
+            resource_class=DatabaseResource,
+            also_requires="view-database",
+        ),
         Action(
             name="create-table",
             abbr="ct",
             description="Create tables",
             resource_class=DatabaseResource,
         ),
+        Action(
+            name="create-view",
+            abbr="cv",
+            description="Create views",
+            resource_class=DatabaseResource,
+        ),
+        Action(
+            name="store-query",
+            abbr="sq",
+            description="Create stored queries",
+            resource_class=DatabaseResource,
+            also_requires="execute-sql",
+        ),
         # Table-level actions (child-level)
         Action(
             name="view-table",
@@ -85,12 +105,24 @@ def register_actions():
             description="Alter tables",
             resource_class=TableResource,
         ),
+        Action(
+            name="set-column-type",
+            abbr="sct",
+            description="Set column type",
+            resource_class=TableResource,
+        ),
         Action(
             name="drop-table",
             abbr="dt",
             description="Drop tables",
             resource_class=TableResource,
         ),
+        Action(
+            name="drop-view",
+            abbr="dv",
+            description="Drop views",
+            resource_class=TableResource,
+        ),
         # Query-level actions (child-level)
         Action(
             name="view-query",
@@ -98,4 +130,16 @@ def register_actions():
             description="View named query results",
             resource_class=QueryResource,
         ),
+        Action(
+            name="update-query",
+            abbr="uq",
+            description="Update stored queries",
+            resource_class=QueryResource,
+        ),
+        Action(
+            name="delete-query",
+            abbr="dq",
+            description="Delete stored queries",
+            resource_class=QueryResource,
+        ),
     )

datasette/default_column_types.py

@@ -0,0 +1,101 @@
+import json
+import re
+
+import markupsafe
+
+from datasette import hookimpl
+from datasette.column_types import ColumnType, SQLiteType
+
+_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE)
+
+
+def _normalize_http_url(value):
+    if not isinstance(value, str):
+        return None
+    normalized = value.strip()
+    if not _HTTP_URL_RE.fullmatch(normalized):
+        return None
+    return normalized
+
+
+class UrlColumnType(ColumnType):
+    name = "url"
+    description = "URL"
+    sqlite_types = (SQLiteType.TEXT,)
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        if not value or not isinstance(value, str):
+            return None
+        normalized = _normalize_http_url(value)
+        if normalized is None:
+            return markupsafe.escape(value.strip())
+        escaped = markupsafe.escape(normalized)
+        return markupsafe.Markup(f'<a href="{escaped}">{escaped}</a>')
+
+    async def validate(self, value, datasette):
+        if value is None or value == "":
+            return None
+        if not isinstance(value, str):
+            return "URL must be a string"
+        if _normalize_http_url(value) is None:
+            return "Invalid URL"
+        return None
+
+
+class EmailColumnType(ColumnType):
+    name = "email"
+    description = "Email address"
+    sqlite_types = (SQLiteType.TEXT,)
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        if not value or not isinstance(value, str):
+            return None
+        escaped = markupsafe.escape(value.strip())
+        return markupsafe.Markup(f'<a href="mailto:{escaped}">{escaped}</a>')
+
+    async def validate(self, value, datasette):
+        if value is None or value == "":
+            return None
+        if not isinstance(value, str):
+            return "Email must be a string"
+        if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", value.strip()):
+            return "Invalid email address"
+        return None
+
+
+class JsonColumnType(ColumnType):
+    name = "json"
+    description = "JSON data"
+    sqlite_types = (SQLiteType.TEXT,)
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        if value is None:
+            return None
+        try:
+            parsed = json.loads(value) if isinstance(value, str) else value
+            formatted = json.dumps(parsed, indent=2)
+            escaped = markupsafe.escape(formatted)
+            return markupsafe.Markup(f"<pre>{escaped}</pre>")
+        except (json.JSONDecodeError, TypeError):
+            return None
+
+    async def validate(self, value, datasette):
+        if value is None or value == "":
+            return None
+        if isinstance(value, str):
+            try:
+                json.loads(value)
+            except json.JSONDecodeError:
+                return "Invalid JSON"
+        return None
+
+
+class TextareaColumnType(ColumnType):
+    name = "textarea"
+    description = "Multiline text"
+    sqlite_types = (SQLiteType.TEXT,)
+
+
+@hookimpl
+def register_column_types(datasette):
+    return [UrlColumnType, EmailColumnType, JsonColumnType, TextareaColumnType]

datasette/default_database_actions.py

@@ -0,0 +1,24 @@
+from datasette import hookimpl
+from datasette.resources import DatabaseResource
+
+
+@hookimpl
+def database_actions(datasette, actor, database, request):
+    async def inner():
+        if not datasette.get_database(database).is_mutable:
+            return []
+        if not await datasette.allowed(
+            action="execute-write-sql",
+            resource=DatabaseResource(database),
+            actor=actor,
+        ):
+            return []
+        return [
+            {
+                "href": datasette.urls.database(database) + "/-/execute-write",
+                "label": "Execute write SQL",
+                "description": "Run writable SQL with table permission checks.",
+            }
+        ]
+
+    return inner

datasette/default_debug_menu.py

@@ -0,0 +1,80 @@
+from datasette import hookimpl
+from datasette.jump import JumpSQL
+
+DEBUG_MENU_ITEMS = (
+    (
+        "/-/databases",
+        "Databases",
+        "List of databases known to this Datasette instance.",
+    ),
+    (
+        "/-/plugins",
+        "Installed plugins",
+        "Review loaded plugins, their versions and their registered hooks.",
+    ),
+    (
+        "/-/versions",
+        "Version info",
+        "Check the Python, SQLite and dependency versions used by this server.",
+    ),
+    (
+        "/-/settings",
+        "Settings",
+        "Inspect the active Datasette settings and configuration values.",
+    ),
+    (
+        "/-/permissions",
+        "Debug permissions",
+        "Test permission checks for actors, actions and resources.",
+    ),
+    (
+        "/-/messages",
+        "Debug messages",
+        "Try out temporary flash messages shown to users.",
+    ),
+    (
+        "/-/allow-debug",
+        "Debug allow rules",
+        "Explore how allow blocks match actors against permission rules.",
+    ),
+    (
+        "/-/debug/autocomplete",
+        "Debug autocomplete",
+        "Try out table autocomplete against a detected label column.",
+    ),
+    (
+        "/-/threads",
+        "Debug threads",
+        "Inspect worker threads and database tasks.",
+    ),
+    (
+        "/-/actor",
+        "Debug actor",
+        "View the actor object for the current signed-in user.",
+    ),
+    (
+        "/-/patterns",
+        "Pattern portfolio",
+        "Browse Datasette UI patterns.",
+    ),
+)
+
+
+@hookimpl
+def jump_items_sql(datasette, actor, request):
+    async def inner():
+        if not await datasette.allowed(action="debug-menu", actor=actor):
+            return []
+
+        return [
+            JumpSQL.menu_item(
+                label=label,
+                url=datasette.urls.path(path),
+                description=description,
+                search_text=f"debug {label} {description}",
+                item_type="debug",
+            )
+            for path, label, description in DEBUG_MENU_ITEMS
+        ]
+
+    return inner

datasette/default_jump_items.py

@@ -0,0 +1,82 @@
+from datasette import hookimpl
+from datasette.jump import JumpSQL
+
+
+@hookimpl
+def jump_items_sql(datasette, actor, request):
+    async def inner():
+        database_sql, database_params = await datasette.allowed_resources_sql(
+            action="view-database", actor=actor
+        )
+        table_sql, table_params = await datasette.allowed_resources_sql(
+            action="view-table", actor=actor
+        )
+        query_sql, query_params = await datasette.allowed_resources_sql(
+            action="view-query", actor=actor
+        )
+        return [
+            JumpSQL(
+                sql=f"""
+                WITH allowed_databases AS (
+                    {database_sql}
+                )
+                SELECT
+                    'database' AS type,
+                    parent AS label,
+                    NULL AS description,
+                    json_object(
+                        'method', 'database',
+                        'database', parent
+                    ) AS url,
+                    parent AS search_text,
+                    NULL AS display_name
+                FROM allowed_databases
+                """,
+                params=database_params,
+            ),
+            JumpSQL(
+                sql=f"""
+                WITH allowed_tables AS (
+                    {table_sql}
+                )
+                SELECT
+                    CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type,
+                    allowed_tables.parent || ': ' || allowed_tables.child AS label,
+                    NULL AS description,
+                    json_object(
+                        'method', 'table',
+                        'database', allowed_tables.parent,
+                        'table', allowed_tables.child
+                    ) AS url,
+                    allowed_tables.parent || ' ' || allowed_tables.child AS search_text,
+                    NULL AS display_name
+                FROM allowed_tables
+                LEFT JOIN catalog_views
+                    ON catalog_views.database_name = allowed_tables.parent
+                   AND catalog_views.view_name = allowed_tables.child
+                """,
+                params=table_params,
+            ),
+            JumpSQL(
+                sql=f"""
+                WITH allowed_queries AS (
+                    {query_sql}
+                )
+                SELECT
+                    'query' AS type,
+                    allowed_queries.parent || ': ' || allowed_queries.child AS label,
+                    NULL AS description,
+                    json_object(
+                        'method', 'query',
+                        'database', allowed_queries.parent,
+                        'query', allowed_queries.child
+                    ) AS url,
+                    allowed_queries.parent || ' ' || allowed_queries.child AS search_text,
+                    NULL AS display_name
+                FROM allowed_queries
+                """,
+                params=query_params,
+            ),
+        ]
+
+    return inner

datasette/default_magic_parameters.py

@@ -1,8 +1,9 @@
-from datasette import hookimpl
 import datetime
 import os
 import time
 
+from datasette import hookimpl
+
 
 def header(key, request):
     key = key.replace("_", "-").encode("utf-8")

datasette/default_menu_links.py

@@ -1,41 +0,0 @@
-from datasette import hookimpl
-
-
-@hookimpl
-def menu_links(datasette, actor):
-    async def inner():
-        if not await datasette.allowed(action="debug-menu", actor=actor):
-            return []
-
-        return [
-            {"href": datasette.urls.path("/-/databases"), "label": "Databases"},
-            {
-                "href": datasette.urls.path("/-/plugins"),
-                "label": "Installed plugins",
-            },
-            {
-                "href": datasette.urls.path("/-/versions"),
-                "label": "Version info",
-            },
-            {
-                "href": datasette.urls.path("/-/settings"),
-                "label": "Settings",
-            },
-            {
-                "href": datasette.urls.path("/-/permissions"),
-                "label": "Debug permissions",
-            },
-            {
-                "href": datasette.urls.path("/-/messages"),
-                "label": "Debug messages",
-            },
-            {
-                "href": datasette.urls.path("/-/allow-debug"),
-                "label": "Debug allow rules",
-            },
-            {"href": datasette.urls.path("/-/threads"), "label": "Debug threads"},
-            {"href": datasette.urls.path("/-/actor"), "label": "Debug actor"},
-            {"href": datasette.urls.path("/-/patterns"), "label": "Pattern portfolio"},
-        ]
-
-    return inner

datasette/default_permissions/__init__.py

@@ -17,43 +17,29 @@ UNION/INTERSECT operations. The order of evaluation is:
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional
+from .config import config_permissions_sql as config_permissions_sql
-
+from .defaults import (
-if TYPE_CHECKING:
+    DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS,
-    from datasette.app import Datasette
+)
-
+from .defaults import (
-from datasette import hookimpl
+    default_action_permissions_sql as default_action_permissions_sql,
+)
+from .defaults import (
+    # Avoid "datasette.default_permissions" does not explicitly export attribute
+    default_allow_sql_check as default_allow_sql_check,
+)
+from .defaults import (
+    default_query_permissions_sql as default_query_permissions_sql,
+)
+from .restrictions import (
+    ActorRestrictions as ActorRestrictions,
+)
 
 # Re-export all hooks and public utilities
 from .restrictions import (
-    actor_restrictions_sql,
+    actor_restrictions_sql as actor_restrictions_sql,
-    restrictions_allow_action,
-    ActorRestrictions,
 )
-from .root import root_user_permissions_sql
+from .restrictions import (
-from .config import config_permissions_sql
+    restrictions_allow_action as restrictions_allow_action,
-from .defaults import (
-    default_allow_sql_check,
-    default_action_permissions_sql,
-    DEFAULT_ALLOW_ACTIONS,
 )
-from .tokens import actor_from_signed_api_token
+from .root import root_user_permissions_sql as root_user_permissions_sql
-
-
-@hookimpl
-def skip_csrf(scope) -> Optional[bool]:
-    """Skip CSRF check for JSON content-type requests."""
-    if scope["type"] == "http":
-        headers = scope.get("headers") or {}
-        if dict(headers).get(b"content-type") == b"application/json":
-            return True
-    return None
-
-
-@hookimpl
-def canned_queries(datasette: "Datasette", database: str, actor) -> dict:
-    """Return canned queries defined in datasette.yaml configuration."""
-    queries = (
-        ((datasette.config or {}).get("databases") or {}).get(database) or {}
-    ).get("queries") or {}
-    return queries

datasette/default_permissions/config.py

@@ -6,7 +6,7 @@ Applies permission rules from datasette.yaml configuration.
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -55,8 +55,8 @@ class ConfigPermissionProcessor:
 
     def __init__(
         self,
-        datasette: "Datasette",
+        datasette: Datasette,
-        actor: Optional[dict],
+        actor: dict | None,
         action: str,
     ):
         self.datasette = datasette
@@ -74,8 +74,8 @@ class ConfigPermissionProcessor:
         self.restrictions = actor.get("_r", {}) if actor else {}
 
         # Pre-compute restriction info for efficiency
-        self.restricted_databases: Set[str] = set()
+        self.restricted_databases: set[str] = set()
-        self.restricted_tables: Set[Tuple[str, str]] = set()
+        self.restricted_tables: set[tuple[str, str]] = set()
 
         if self.has_restrictions:
             self.restricted_databases = {
@@ -92,16 +92,27 @@ class ConfigPermissionProcessor:
             # Tables implicitly reference their parent databases
             self.restricted_databases.update(db for db, _ in self.restricted_tables)
 
-    def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]:
+        # Resolve identity keys once per action, rather than scanning the
+        # restriction allowlist for every configured table's allow block.
+        self.restricted_table_keys = {
+            (db, self.action_obj.normalize_child(table) if self.action_obj else table)
+            for db, table in self.restricted_tables
+        }
+
+    def evaluate_allow_block(self, allow_block: Any) -> bool | None:
         """Evaluate an allow block against the current actor."""
         if allow_block is None:
             return None
+        # Values passed using ``-s permissions.* 1`` or ``0`` are parsed as
+        # integers, but should retain the CLI's boolean 1/0 behavior.
+        if isinstance(allow_block, int) and allow_block in (0, 1):
+            return bool(allow_block)
         return actor_matches_allow(self.actor, allow_block)
 
     def is_in_restriction_allowlist(
         self,
-        parent: Optional[str],
+        parent: str | None,
-        child: Optional[str],
+        child: str | None,
     ) -> bool:
         """Check if resource is allowed by actor restrictions."""
         if not self.has_restrictions:
@@ -121,8 +132,10 @@ class ConfigPermissionProcessor:
         if parent:
             table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {})
             if child:
-                table_actions = table_restrictions.get(child, [])
+                child_key = (
-                if self.action_checks.intersection(table_actions):
+                    self.action_obj.normalize_child(child) if self.action_obj else child
+                )
+                if (parent, child_key) in self.restricted_table_keys:
                     return True
             else:
                 # Parent query should proceed if any child in this database is allowlisted
@@ -143,9 +156,9 @@ class ConfigPermissionProcessor:
 
     def add_permissions_rule(
         self,
-        parent: Optional[str],
+        parent: str | None,
-        child: Optional[str],
+        child: str | None,
-        permissions_block: Optional[dict],
+        permissions_block: dict | None,
         scope_desc: str,
     ) -> None:
         """Add a rule from a permissions:{action} block."""
@@ -165,8 +178,8 @@ class ConfigPermissionProcessor:
 
     def add_allow_block_rule(
         self,
-        parent: Optional[str],
+        parent: str | None,
-        child: Optional[str],
+        child: str | None,
         allow_block: Any,
         scope_desc: str,
     ) -> None:
@@ -198,8 +211,8 @@ class ConfigPermissionProcessor:
 
     def _add_restriction_gate_denies(
         self,
-        parent: Optional[str],
+        parent: str | None,
-        child: Optional[str],
+        child: str | None,
         is_allowed: bool,
         scope_desc: str,
     ) -> None:
@@ -231,7 +244,7 @@ class ConfigPermissionProcessor:
                     if db_name == parent:
                         self.collector.add(db_name, table_name, False, reason)
 
-    def process(self) -> Optional[PermissionSQL]:
+    def process(self) -> PermissionSQL | None:
         """Process all config rules and return combined PermissionSQL."""
         self._process_root_permissions()
         self._process_databases()
@@ -421,10 +434,10 @@ class ConfigPermissionProcessor:
 
 @hookimpl(specname="permission_resources_sql")
 async def config_permissions_sql(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
     action: str,
-) -> Optional[List[PermissionSQL]]:
+) -> list[PermissionSQL] | None:
     """
     Apply permission rules from datasette.yaml configuration.
 

datasette/default_permissions/defaults.py

@@ -6,7 +6,7 @@ Provides default allow rules for standard view/execute actions.
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -14,7 +14,6 @@ if TYPE_CHECKING:
 from datasette import hookimpl
 from datasette.permissions import PermissionSQL
 
-
 # Actions that are allowed by default (unless --default-deny is used)
 DEFAULT_ALLOW_ACTIONS = frozenset(
     {
@@ -30,18 +29,17 @@ DEFAULT_ALLOW_ACTIONS = frozenset(
 
 @hookimpl(specname="permission_resources_sql")
 async def default_allow_sql_check(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
     action: str,
-) -> Optional[PermissionSQL]:
+) -> PermissionSQL | None:
     """
     Enforce the default_allow_sql setting.
 
     When default_allow_sql is false (the default), execute-sql is denied
     unless explicitly allowed by config or other rules.
     """
-    if action == "execute-sql":
+    if action == "execute-sql" and not datasette.setting("default_allow_sql"):
-        if not datasette.setting("default_allow_sql"):
         return PermissionSQL.deny(reason="default_allow_sql is false")
 
     return None
@@ -49,10 +47,10 @@ async def default_allow_sql_check(
 
 @hookimpl(specname="permission_resources_sql")
 async def default_action_permissions_sql(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
     action: str,
-) -> Optional[PermissionSQL]:
+) -> PermissionSQL | None:
     """
     Provide default allow rules for standard view/execute actions.
 
@@ -68,3 +66,48 @@ async def default_action_permissions_sql(
         return PermissionSQL.allow(reason=reason)
 
     return None
+
+
+@hookimpl(specname="permission_resources_sql")
+async def default_query_permissions_sql(
+    datasette: Datasette,
+    actor: dict | None,
+    action: str,
+) -> PermissionSQL | None:
+    actor_id = actor.get("id") if isinstance(actor, dict) else None
+
+    if action not in {"view-query", "update-query", "delete-query"}:
+        return None
+
+    params = {"query_owner_id": actor_id}
+    rule_sqls = []
+    if actor_id is not None:
+        if action in {"update-query", "delete-query"}:
+            # Query owner can update/delete query
+            rule_sqls.append("""
+            SELECT database_name AS parent, name AS child, 1 AS allow,
+              'query owner' AS reason
+            FROM queries
+            WHERE source = 'user'
+              AND owner_id = :query_owner_id
+            """)
+        else:
+            # Query owner can view-query
+            rule_sqls.append("""
+            SELECT database_name AS parent, name AS child, 1 AS allow,
+              'query owner' AS reason
+            FROM queries
+            WHERE owner_id = :query_owner_id
+            """)
+
+    # restriction_sql enforces private queries ONLY visible/mutable by owner
+    return PermissionSQL(
+        sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None,
+        restriction_sql="""
+        SELECT database_name AS parent, name AS child
+        FROM queries
+        WHERE is_private = 0
+           OR owner_id = :query_owner_id
+        """,
+        params=params,
+    )

datasette/default_permissions/helpers.py

@@ -5,7 +5,7 @@ Shared helper utilities for default permission implementations.
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List, Optional, Set
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -13,7 +13,7 @@ if TYPE_CHECKING:
 from datasette.permissions import PermissionSQL
 
 
-def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]:
+def get_action_name_variants(datasette: Datasette, action: str) -> set[str]:
     """
     Get all name variants for an action (full name and abbreviation).
 
@@ -27,7 +27,7 @@ def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]:
     return variants
 
 
-def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool:
+def action_in_list(datasette: Datasette, action: str, action_list: list) -> bool:
     """Check if an action (or its abbreviation) is in a list."""
     return bool(get_action_name_variants(datasette, action).intersection(action_list))
 
@@ -36,8 +36,8 @@ def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bo
 class PermissionRow:
     """A single permission rule row."""
 
-    parent: Optional[str]
+    parent: str | None
-    child: Optional[str]
+    child: str | None
     allow: bool
     reason: str
 
@@ -46,14 +46,14 @@ class PermissionRowCollector:
     """Collects permission rows and converts them to PermissionSQL."""
 
     def __init__(self, prefix: str = "row"):
-        self.rows: List[PermissionRow] = []
+        self.rows: list[PermissionRow] = []
         self.prefix = prefix
 
     def add(
         self,
-        parent: Optional[str],
+        parent: str | None,
-        child: Optional[str],
+        child: str | None,
-        allow: Optional[bool],
+        allow: bool | None,
         reason: str,
         if_not_none: bool = False,
     ) -> None:
@@ -62,7 +62,7 @@ class PermissionRowCollector:
             return
         self.rows.append(PermissionRow(parent, child, allow, reason))
 
-    def to_permission_sql(self) -> Optional[PermissionSQL]:
+    def to_permission_sql(self) -> PermissionSQL | None:
         """Convert collected rows to a PermissionSQL object."""
         if not self.rows:
             return None

datasette/default_permissions/restrictions.py

@@ -8,7 +8,7 @@ contains allowlists of resources the actor can access.
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, List, Optional, Set, Tuple
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -23,12 +23,12 @@ from .helpers import action_in_list, get_action_name_variants
 class ActorRestrictions:
     """Parsed actor restrictions from the _r key."""
 
-    global_actions: List[str]  # _r.a - globally allowed actions
+    global_actions: list[str]  # _r.a - globally allowed actions
     database_actions: dict  # _r.d - {db_name: [actions]}
     table_actions: dict  # _r.r - {db_name: {table: [actions]}}
 
     @classmethod
-    def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]:
+    def from_actor(cls, actor: dict | None) -> ActorRestrictions | None:
         """Parse restrictions from actor dict. Returns None if no restrictions."""
         if not actor:
             return None
@@ -44,11 +44,11 @@ class ActorRestrictions:
             table_actions=restrictions.get("r", {}),
         )
 
-    def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool:
+    def is_action_globally_allowed(self, datasette: Datasette, action: str) -> bool:
         """Check if action is in the global allowlist."""
         return action_in_list(datasette, action, self.global_actions)
 
-    def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]:
+    def get_allowed_databases(self, datasette: Datasette, action: str) -> set[str]:
         """Get database names where this action is allowed."""
         allowed = set()
         for db_name, db_actions in self.database_actions.items():
@@ -57,8 +57,8 @@ class ActorRestrictions:
         return allowed
 
     def get_allowed_tables(
-        self, datasette: "Datasette", action: str
+        self, datasette: Datasette, action: str
-    ) -> Set[Tuple[str, str]]:
+    ) -> set[tuple[str, str]]:
         """Get (database, table) pairs where this action is allowed."""
         allowed = set()
         for db_name, tables in self.table_actions.items():
@@ -70,10 +70,10 @@ class ActorRestrictions:
 
 @hookimpl(specname="permission_resources_sql")
 async def actor_restrictions_sql(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
     action: str,
-) -> Optional[List[PermissionSQL]]:
+) -> list[PermissionSQL] | None:
     """
     Handle actor restriction-based permission rules.
 
@@ -140,10 +140,10 @@ async def actor_restrictions_sql(
 
 
 def restrictions_allow_action(
-    datasette: "Datasette",
+    datasette: Datasette,
     restrictions: dict,
     action: str,
-    resource: Optional[str | Tuple[str, str]],
+    resource: str | tuple[str, str] | None,
 ) -> bool:
     """
     Check if restrictions allow the requested action on the requested resource.
@@ -185,8 +185,12 @@ def restrictions_allow_action(
     # Check table/resource level
     if resource is not None and not isinstance(resource, str) and len(resource) == 2:
         database, table = resource
-        table_allowed = restrictions.get("r", {}).get(database, {}).get(table)
+        action_obj = datasette.actions.get(action)
-        if table_allowed is not None:
+        normalize = action_obj.normalize_child if action_obj else lambda name: name
+        for table_name, table_allowed in (
+            restrictions.get("r", {}).get(database, {}).items()
+        ):
+            if normalize(table_name) == normalize(table):
                 assert isinstance(table_allowed, list)
                 if to_check.intersection(table_allowed):
                     return True

datasette/default_permissions/root.py

@@ -6,7 +6,7 @@ Grants full permissions to the root user when --root flag is used.
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Optional
+from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
@@ -17,9 +17,9 @@ from datasette.permissions import PermissionSQL
 
 @hookimpl(specname="permission_resources_sql")
 async def root_user_permissions_sql(
-    datasette: "Datasette",
+    datasette: Datasette,
-    actor: Optional[dict],
+    actor: dict | None,
-) -> Optional[PermissionSQL]:
+) -> PermissionSQL | None:
     """
     Grant root user full permissions when --root flag is used.
     """

datasette/default_permissions/sqlite_statistics.py

@@ -0,0 +1,25 @@
+"""Default table-access policy for SQLite optimizer statistics."""
+
+import json
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+
+@hookimpl
+def permission_resources_sql(action):
+    if action != "view-table":
+        return None
+    return PermissionSQL(
+        sql="""
+            SELECT database_name AS parent, value AS child, 0 AS allow,
+                'SQLite statistics tables are denied by default' AS reason
+            FROM catalog_databases
+            CROSS JOIN json_each(:sqlite_statistics_names)
+        """,
+        params={
+            "sqlite_statistics_names": json.dumps(
+                ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"]
+            )
+        },
+    )

datasette/default_permissions/tokens.py

@@ -1,44 +1,33 @@
 """
 Token authentication for Datasette.
 
-Handles signed API tokens (dstok_ prefix).
+Registers the default SignedTokenHandler and delegates token verification
+to datasette.verify_token() so all registered handlers are tried.
 """
 
 from __future__ import annotations
 
-import time
+from typing import TYPE_CHECKING
-from typing import TYPE_CHECKING, Optional
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
 
-import itsdangerous
-
 from datasette import hookimpl
+from datasette.tokens import SignedTokenHandler
+
+
+@hookimpl
+def register_token_handler(datasette: Datasette):
+    """Register the default signed token handler."""
+    return SignedTokenHandler()
 
 
 @hookimpl(specname="actor_from_request")
-def actor_from_signed_api_token(datasette: "Datasette", request) -> Optional[dict]:
+async def actor_from_signed_api_token(datasette: Datasette, request) -> dict | None:
     """
-    Authenticate requests using signed API tokens (dstok_ prefix).
+    Authenticate requests using API tokens by delegating to all registered
-
+    token handlers via datasette.verify_token().
-    Token structure (signed JSON):
-    {
-        "a": "actor_id",      # Actor ID
-        "t": 1234567890,      # Timestamp (Unix epoch)
-        "d": 3600,            # Optional: Duration in seconds
-        "_r": {...}           # Optional: Restrictions
-    }
     """
-    prefix = "dstok_"
-
-    # Check if tokens are enabled
-    if not datasette.setting("allow_signed_tokens"):
-        return None
-
-    max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl")
-
-    # Get authorization header
     authorization = request.headers.get("authorization")
     if not authorization:
         return None
@@ -46,50 +35,4 @@ def actor_from_signed_api_token(datasette: "Datasette", request) -> Optional[dic
         return None
 
     token = authorization[len("Bearer ") :]
-    if not token.startswith(prefix):
+    return await datasette.verify_token(token)
-        return None
-
-    # Remove prefix and verify signature
-    token = token[len(prefix) :]
-    try:
-        decoded = datasette.unsign(token, namespace="token")
-    except itsdangerous.BadSignature:
-        return None
-
-    # Validate timestamp
-    if "t" not in decoded:
-        return None
-    created = decoded["t"]
-    if not isinstance(created, int):
-        return None
-
-    # Handle duration/expiry
-    duration = decoded.get("d")
-    if duration is not None and not isinstance(duration, int):
-        return None
-
-    # Apply max TTL if configured
-    if (duration is None and max_signed_tokens_ttl) or (
-        duration is not None
-        and max_signed_tokens_ttl
-        and duration > max_signed_tokens_ttl
-    ):
-        duration = max_signed_tokens_ttl
-
-    # Check expiry
-    if duration:
-        if time.time() - created > duration:
-            return None
-
-    # Build actor dict
-    actor = {"id": decoded["a"], "token": "dstok"}
-
-    # Copy restrictions if present
-    if "_r" in decoded:
-        actor["_r"] = decoded["_r"]
-
-    # Add expiry timestamp if applicable
-    if duration:
-        actor["token_expires"] = created + duration
-
-    return actor

datasette/default_query_actions.py

@@ -0,0 +1,48 @@
+from datasette import hookimpl
+from datasette.resources import QueryResource
+
+
+@hookimpl
+def query_actions(datasette, actor, database, query_name, request):
+    # Only stored queries (with a name) can be edited or deleted
+    if not query_name:
+        return None
+
+    async def inner():
+        query = await datasette.get_query(database, query_name)
+        if query is None:
+            return []
+        # Config-defined and trusted queries are managed outside the UI
+        if query.source == "config" or query.is_trusted:
+            return []
+
+        links = []
+        if await datasette.allowed(
+            action="update-query",
+            resource=QueryResource(database, query_name),
+            actor=actor,
+        ):
+            links.append(
+                {
+                    "href": datasette.urls.table(database, query_name) + "/-/edit",
+                    "label": "Edit this query",
+                    "description": (
+                        "Change the title, description, SQL or visibility."
+                    ),
+                }
+            )
+        if await datasette.allowed(
+            action="delete-query",
+            resource=QueryResource(database, query_name),
+            actor=actor,
+        ):
+            links.append(
+                {
+                    "href": datasette.urls.table(database, query_name) + "/-/delete",
+                    "label": "Delete this query",
+                    "description": "Permanently remove this saved query.",
+                }
+            )
+        return links
+
+    return inner

datasette/default_table_actions.py

@@ -0,0 +1,29 @@
+from datasette import hookimpl
+from datasette.resources import TableResource
+
+
+@hookimpl
+def table_actions(datasette, actor, database, table, request):
+    async def inner():
+        db = datasette.get_database(database)
+        if not db.is_mutable:
+            return []
+        if not await datasette.allowed(
+            action="alter-table",
+            resource=TableResource(database=database, table=table),
+            actor=actor,
+        ):
+            return []
+        return [
+            {
+                "type": "button",
+                "label": "Alter table",
+                "description": "Change columns and primary key for this table.",
+                "attrs": {
+                    "aria-label": f"Alter table {table}",
+                    "data-table-action": "alter-table",
+                },
+            }
+        ]
+
+    return inner

datasette/events.py

@@ -1,8 +1,9 @@
 from abc import ABC, abstractproperty
 from dataclasses import asdict, dataclass, field
-from datasette.hookspecs import hookimpl
 from datetime import datetime, timezone
 
+from datasette.hookspecs import hookimpl
+
 
 @dataclass
 class Event(ABC):
@@ -199,6 +200,27 @@ class UpdateRowEvent(Event):
     pks: list
 
 
+@dataclass
+class RenameTableEvent(Event):
+    """
+    Event name: ``rename-table``
+
+    A table has been renamed.
+
+    :ivar database: The name of the database containing the renamed table.
+    :type database: str
+    :ivar old_table: The previous name of the table.
+    :type old_table: str
+    :ivar new_table: The new name of the table.
+    :type new_table: str
+    """
+
+    name = "rename-table"
+    database: str
+    old_table: str
+    new_table: str
+
+
 @dataclass
 class DeleteRowEvent(Event):
     """
@@ -219,6 +241,42 @@ class DeleteRowEvent(Event):
     pks: list
 
 
+@hookimpl
+def write_wrapper(datasette, database, request, transaction):
+    def wrapper(conn, track_event):
+        # Snapshot rootpage -> name before the write
+        before = {
+            row[1]: row[0]
+            for row in conn.execute(
+                "select name, rootpage from sqlite_master"
+                " where type='table' and rootpage != 0"
+            ).fetchall()
+        }
+        yield
+        # Snapshot rootpage -> name after the write
+        after = {
+            row[1]: row[0]
+            for row in conn.execute(
+                "select name, rootpage from sqlite_master"
+                " where type='table' and rootpage != 0"
+            ).fetchall()
+        }
+        # Detect renames: same rootpage, different name
+        for rootpage, old_name in before.items():
+            new_name = after.get(rootpage)
+            if new_name and new_name != old_name:
+                track_event(
+                    RenameTableEvent(
+                        actor=request.actor if request else None,
+                        database=database,
+                        old_table=old_name,
+                        new_table=new_name,
+                    )
+                )
+
+    return wrapper
+
+
 @hookimpl
 def register_events():
     return [
@@ -227,6 +285,7 @@ def register_events():
         CreateTableEvent,
         CreateTokenEvent,
         AlterTableEvent,
+        RenameTableEvent,
         DropTableEvent,
         InsertRowsEvent,
         UpsertRowsEvent,

datasette/extras.py

@@ -0,0 +1,141 @@
+import re
+from dataclasses import dataclass
+from enum import Enum
+from typing import ClassVar
+
+from asyncinject import Registry
+
+from datasette.utils.asgi import BadRequest
+
+
+def extra_names_from_request(request):
+    extra_bits = request.args.getlist("_extra")
+    extras = set()
+    for bit in extra_bits:
+        extras.update(part for part in bit.split(",") if part)
+    return extras
+
+
+class ExtraScope(Enum):
+    TABLE = "table"
+    ROW = "row"
+    QUERY = "query"
+
+
+@dataclass(frozen=True)
+class ExtraExample:
+    path: str | None = None
+    key: str | None = None
+    value: object | None = None
+    note: str | None = None
+
+
+class Provider:
+    name: ClassVar[str | None] = None
+    scopes: ClassVar[set[ExtraScope]] = set()
+    public: ClassVar[bool] = False
+
+    @classmethod
+    def key(cls):
+        return cls.name or _camel_to_snake(cls.__name__)
+
+    @classmethod
+    def available_for(cls, scope):
+        return scope in cls.scopes
+
+    async def resolve(self, context):
+        raise NotImplementedError
+
+
+class Extra(Provider):
+    description: ClassVar[str | None] = None
+    example: ClassVar[ExtraExample | None] = None
+    examples: ClassVar[dict[ExtraScope, ExtraExample | list[ExtraExample]]] = {}
+    public: ClassVar[bool] = True
+    expensive: ClassVar[bool] = False
+    docs_note: ClassVar[str | None] = None
+
+    @classmethod
+    def example_for_scope(cls, scope):
+        return cls.examples.get(scope, cls.example)
+
+
+class ExtraRegistry:
+    def __init__(self, classes):
+        self.classes = list(classes)
+        self.classes_by_name = {cls.key(): cls for cls in self.classes}
+        # Lazily-built shared state, keyed by scope. Safe to share across
+        # requests because Extra instances are stateless and asyncinject's
+        # Registry keeps per-call state local to each resolve_multi() call.
+        # If extras classes ever become registerable at runtime (e.g. via a
+        # plugin hook) these caches will need invalidating.
+        self._scope_registries = {}
+        self._allowed_names = {}
+
+    def classes_for_scope(self, scope, include_internal=True):
+        classes = [
+            cls
+            for cls in self.classes
+            if cls.available_for(scope) and (include_internal or cls.public)
+        ]
+        return classes
+
+    def public_classes_for_scope(self, scope):
+        return self.classes_for_scope(scope, include_internal=False)
+
+    def internal_classes_for_scope(self, scope):
+        # Extras that are available to HTML templates but excluded from
+        # JSON responses - plain Providers are dependency plumbing and
+        # never surface as keys, so they are not included
+        return [
+            cls
+            for cls in self.classes_for_scope(scope)
+            if issubclass(cls, Extra) and not cls.public
+        ]
+
+    def _registry_for_scope(self, scope):
+        registry = self._scope_registries.get(scope)
+        if registry is None:
+            registry = Registry()
+            for cls in self.classes_for_scope(scope):
+                registry.register(cls().resolve, name=cls.key())
+            self._scope_registries[scope] = registry
+        return registry
+
+    def _allowed_names_for_scope(self, scope, include_internal):
+        key = (scope, include_internal)
+        names = self._allowed_names.get(key)
+        if names is None:
+            names = {
+                cls.key()
+                for cls in self.classes_for_scope(
+                    scope, include_internal=include_internal
+                )
+            }
+            self._allowed_names[key] = names
+        return names
+
+    def validate_requested(self, requested, scope):
+        """
+        Raise BadRequest if any requested extra name is not a public extra
+        for this scope. Used by data formats such as .json - HTML pages
+        silently ignore unknown names instead.
+        """
+        allowed = self._allowed_names_for_scope(scope, include_internal=False)
+        unknown = sorted(name for name in requested if name not in allowed)
+        if unknown:
+            raise BadRequest("Unknown _extra: {}".format(", ".join(unknown)))
+
+    async def resolve(self, requested, context, scope, include_internal=False):
+        allowed_names = self._allowed_names_for_scope(scope, include_internal)
+        requested_names = [name for name in requested if name in allowed_names]
+        resolved = await self._registry_for_scope(scope).resolve_multi(
+            requested_names, results={"context": context}
+        )
+        return {name: resolved[name] for name in requested_names}
+
+
+def _camel_to_snake(name):
+    name = re.sub(r"(Extra|Provider)$", "", name)
+    name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name)
+    return re.sub("([a-z0-9])([A-Z])", r"\1_\2", name).lower()

datasette/facets.py

@@ -1,12 +1,13 @@
 import json
 import urllib
+
 from datasette import hookimpl
 from datasette.database import QueryInterrupted
 from datasette.utils import (
+    detect_json1,
     escape_sqlite,
     path_with_added_args,
     path_with_removed_args,
-    detect_json1,
     sqlite3,
 )
 
@@ -30,7 +31,7 @@ def load_facet_configs(request, table_config):
             assert (
                 len(facet_config.values()) == 1
             ), "Metadata config dicts should be {type: config}"
-            type, facet_config = list(facet_config.items())[0]
+            type, facet_config = next(iter(facet_config.items()))
             if isinstance(facet_config, str):
                 facet_config = {"simple": facet_config}
         facet_configs.setdefault(type, []).append(
@@ -38,7 +39,7 @@ def load_facet_configs(request, table_config):
         )
     qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True)
     for key, values in qs_pairs.items():
-        if key.startswith("_facet"):
+        if key == "_facet" or key.startswith("_facet_"):
             # Figure out the facet type
             if key == "_facet":
                 type = "column"
@@ -83,9 +84,9 @@ class Facet:
         self.ds = ds
         self.request = request
         self.database = database
-        # For foreign key expansion. Can be None for e.g. canned SQL queries:
+        # For foreign key expansion. Can be None for e.g. stored SQL queries:
         self.table = table
-        self.sql = sql or f"select * from [{table}]"
+        self.sql = sql or f"select * from {escape_sqlite(table)}"
         self.params = params or []
         self.table_config = table_config
         # row_count can be None, in which case we calculate it ourselves:
@@ -160,18 +161,13 @@ class ColumnFacet(Facet):
         for column in columns:
             if column in already_enabled:
                 continue
-            suggested_facet_sql = """
+            suggested_facet_sql = f"""
-                with limited as (select * from ({sql}) limit {suggest_consider})
+                with limited as (select * from ({self.sql}) limit {self.suggest_consider})
-                select {column} as value, count(*) as n from limited
+                select {escape_sqlite(column)} as value, count(*) as n from limited
                 where value is not null
                 group by value
-                limit {limit}
+                limit {facet_size + 1}
-            """.format(
+            """
-                column=escape_sqlite(column),
-                sql=self.sql,
-                limit=facet_size + 1,
-                suggest_consider=self.suggest_consider,
-            )
             distinct_values = None
             try:
                 distinct_values = await self.ds.execute(
@@ -233,9 +229,7 @@ class ColumnFacet(Facet):
                 )
                 where {col} is not null
                 group by {col} order by count desc, value limit {limit}
-            """.format(
+            """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1)
-                col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
-            )
             try:
                 facet_rows_results = await self.ds.execute(
                     self.database,
@@ -269,11 +263,16 @@ class ColumnFacet(Facet):
                 for row in facet_rows:
                     column_qs = column
                     if column.startswith("_"):
-                        column_qs = "{}__exact".format(column)
+                        column_qs = f"{column}__exact"
-                    selected = (column_qs, str(row["value"])) in qs_pairs
+                    selected_args = {
+                        key: str(row["value"])
+                        for key in (column_qs, f"{column}__exact")
+                        if (key, str(row["value"])) in qs_pairs
+                    }
+                    selected = bool(selected_args)
                     if selected:
                         toggle_path = path_with_removed_args(
-                            self.request, {column_qs: str(row["value"])}
+                            self.request, selected_args
                         )
                     else:
                         toggle_path = path_with_added_args(
@@ -344,12 +343,12 @@ class ArrayFacet(Facet):
                         for v in await self.ds.execute(
                             self.database,
                             (
-                                "select {column} from ({sql}) "
+                                f"select {escape_sqlite(column)} from ({self.sql}) "
-                                "where {column} is not null "
+                                f"where {escape_sqlite(column)} is not null "
-                                "and {column} != '' "
+                                f"and {escape_sqlite(column)} != '' "
-                                "and json_array_length({column}) > 0 "
+                                f"and json_array_length({escape_sqlite(column)}) > 0 "
                                 "limit 100"
-                            ).format(column=escape_sqlite(column), sql=self.sql),
+                            ),
                             self.params,
                             truncate=False,
                             custom_time_limit=self.ds.setting(
@@ -390,14 +389,14 @@ class ArrayFacet(Facet):
             source = source_and_config["source"]
             column = config.get("column") or config["simple"]
             # https://github.com/simonw/datasette/issues/448
-            facet_sql = """
+            facet_sql = f"""
-                with inner as ({sql}),
+                with inner as ({self.sql}),
                 deduped_array_items as (
                     select
                         distinct j.value,
                         inner.*
                     from
-                        json_each([inner].{col}) j
+                        json_each([inner].{escape_sqlite(column)}) j
                         join inner
                 )
                 select
@@ -408,12 +407,8 @@ class ArrayFacet(Facet):
                 group by
                     value
                 order by
-                    count(*) desc, value limit {limit}
+                    count(*) desc, value limit {facet_size + 1}
-            """.format(
+            """
-                col=escape_sqlite(column),
-                sql=self.sql,
-                limit=facet_size + 1,
-            )
             try:
                 facet_rows_results = await self.ds.execute(
                     self.database,
@@ -482,9 +477,7 @@ class DateFacet(Facet):
                 select date({column}) from (
                     select * from ({sql}) limit 100
                 ) where {column} glob "????-??-*"
-            """.format(
+            """.format(column=escape_sqlite(column), sql=self.sql)
-                column=escape_sqlite(column), sql=self.sql
-            )
             try:
                 results = await self.ds.execute(
                     self.database,
@@ -530,9 +523,7 @@ class DateFacet(Facet):
                 )
                 where date({col}) is not null
                 group by date({col}) order by count desc, value limit {limit}
-            """.format(
+            """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1)
-                col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
-            )
             try:
                 facet_rows_results = await self.ds.execute(
                     self.database,

datasette/filters.py

@@ -1,8 +1,12 @@
-from datasette import hookimpl
-from datasette.resources import DatabaseResource
-from datasette.views.base import DatasetteError
-from datasette.utils.asgi import BadRequest
 import json
+import math
+from typing import ClassVar
+
+from datasette import hookimpl
+from datasette.resources import DatabaseResource, TableResource
+from datasette.utils.asgi import BadRequest
+from datasette.views.base import DatasetteError
+
 from .utils import detect_json1, escape_sqlite, path_with_removed_args
 
 
@@ -48,13 +52,20 @@ def search_filters(request, database, table, datasette):
         human_descriptions = []
         extra_context = {}
 
-        # Figure out which fts_table to use
+        # Figure out which trusted fts_table to use. Query string parameters can
+        # repeat this mapping (for backwards compatibility), but must not select
+        # a different table or primary key.
         table_metadata = await datasette.table_config(database, table)
         db = datasette.get_database(database)
-        fts_table = request.args.get("_fts_table")
+        fts_table = table_metadata.get("fts_table")
-        fts_table = fts_table or table_metadata.get("fts_table")
         fts_table = fts_table or await db.fts_table(table)
-        fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid"))
+        fts_pk = table_metadata.get("fts_pk", "rowid")
+        requested_fts_table = request.args.get("_fts_table")
+        requested_fts_pk = request.args.get("_fts_pk")
+        if (requested_fts_table and requested_fts_table != fts_table) or (
+            requested_fts_pk and requested_fts_pk != fts_pk
+        ):
+            raise BadRequest("Invalid _fts_table or _fts_pk")
         search_args = {
             key: request.args[key]
             for key in request.args
@@ -72,6 +83,11 @@ def search_filters(request, database, table, datasette):
         extra_context["supports_search"] = bool(fts_table)
 
         if fts_table and search_args:
+            await datasette.ensure_permission(
+                action="view-table",
+                resource=TableResource(database=database, table=fts_table),
+                actor=request.actor,
+            )
             if "_search" in search_args:
                 # Simple ?_search=xxx
                 search = search_args["_search"]
@@ -99,9 +115,9 @@ def search_filters(request, database, table, datasette):
                             fts_table=escape_sqlite(fts_table),
                             search_col=escape_sqlite(search_col),
                             match_clause=(
-                                ":search_{}".format(i)
+                                f":search_{i}"
                                 if search_mode_raw
-                                else "escape_fts(:search_{})".format(i)
+                                else f"escape_fts(:search_{i})"
                             ),
                         )
                     )
@@ -132,13 +148,18 @@ def through_filters(request, database, table, datasette):
                 through_table = through_data["table"]
                 other_column = through_data["column"]
                 value = through_data["value"]
+                await datasette.ensure_permission(
+                    action="view-table",
+                    resource=TableResource(database=database, table=through_table),
+                    actor=request.actor,
+                )
                 db = datasette.get_database(database)
                 outgoing_foreign_keys = await db.foreign_keys_for_table(through_table)
-                try:
+                fk_to_us = next(
-                    fk_to_us = [
+                    (fk for fk in outgoing_foreign_keys if fk["other_table"] == table),
-                        fk for fk in outgoing_foreign_keys if fk["other_table"] == table
+                    None,
-                    ][0]
+                )
-                except IndexError:
+                if fk_to_us is None:
                     raise DatasetteError(
                         "Invalid _through - could not find corresponding foreign key"
                     )
@@ -182,6 +203,17 @@ class Filter:
         raise NotImplementedError
 
 
+def _coerce_numeric_filter_value(value):
+    try:
+        return int(value)
+    except ValueError:
+        try:
+            converted = float(value)
+        except ValueError:
+            return value
+        return converted if math.isfinite(converted) else value
+
+
 class TemplatedFilter(Filter):
     def __init__(
         self,
@@ -203,13 +235,17 @@ class TemplatedFilter(Filter):
 
     def where_clause(self, table, column, value, param_counter):
         converted = self.format.format(value)
-        if self.numeric and converted.isdigit():
+        if self.numeric:
-            converted = int(converted)
+            converted = _coerce_numeric_filter_value(converted)
         if self.no_argument:
-            kwargs = {"c": column}
+            kwargs = {"c": _quote_sqlite_identifier(column)}
             converted = None
         else:
-            kwargs = {"c": column, "p": f"p{param_counter}", "t": table}
+            kwargs = {
+                "c": _quote_sqlite_identifier(column),
+                "p": f"p{param_counter}",
+                "t": _quote_sqlite_identifier(table),
+            }
         return self.sql_template.format(**kwargs), converted
 
     def human_clause(self, column, value):
@@ -223,6 +259,14 @@ class TemplatedFilter(Filter):
             return template.format(c=column, v=value)
 
 
+def _quote_sqlite_identifier(identifier):
+    # Preserve the historic always-quoted SQL generated by TemplatedFilter.
+    escaped = escape_sqlite(identifier)
+    if escaped == identifier:
+        return f'"{identifier}"'
+    return escaped
+
+
 class InFilter(Filter):
     key = "in"
     display = "in"
@@ -264,56 +308,56 @@ class Filters:
             TemplatedFilter(
                 "exact",
                 "=",
-                '"{c}" = :{p}',
+                "{c} = :{p}",
                 lambda c, v: "{c} = {v}" if v.isdigit() else '{c} = "{v}"',
             ),
             TemplatedFilter(
                 "not",
                 "!=",
-                '"{c}" != :{p}',
+                "{c} != :{p}",
                 lambda c, v: "{c} != {v}" if v.isdigit() else '{c} != "{v}"',
             ),
             TemplatedFilter(
                 "contains",
                 "contains",
-                '"{c}" like :{p}',
+                "{c} like :{p}",
                 '{c} contains "{v}"',
                 format="%{}%",
             ),
             TemplatedFilter(
                 "notcontains",
                 "does not contain",
-                '"{c}" not like :{p}',
+                "{c} not like :{p}",
                 '{c} does not contain "{v}"',
                 format="%{}%",
             ),
             TemplatedFilter(
                 "endswith",
                 "ends with",
-                '"{c}" like :{p}',
+                "{c} like :{p}",
                 '{c} ends with "{v}"',
                 format="%{}",
             ),
             TemplatedFilter(
                 "startswith",
                 "starts with",
-                '"{c}" like :{p}',
+                "{c} like :{p}",
                 '{c} starts with "{v}"',
                 format="{}%",
             ),
-            TemplatedFilter("gt", ">", '"{c}" > :{p}', "{c} > {v}", numeric=True),
+            TemplatedFilter("gt", ">", "{c} > :{p}", "{c} > {v}", numeric=True),
             TemplatedFilter(
-                "gte", "\u2265", '"{c}" >= :{p}', "{c} \u2265 {v}", numeric=True
+                "gte", "\u2265", "{c} >= :{p}", "{c} \u2265 {v}", numeric=True
             ),
-            TemplatedFilter("lt", "<", '"{c}" < :{p}', "{c} < {v}", numeric=True),
+            TemplatedFilter("lt", "<", "{c} < :{p}", "{c} < {v}", numeric=True),
             TemplatedFilter(
-                "lte", "\u2264", '"{c}" <= :{p}', "{c} \u2264 {v}", numeric=True
+                "lte", "\u2264", "{c} <= :{p}", "{c} \u2264 {v}", numeric=True
             ),
-            TemplatedFilter("like", "like", '"{c}" like :{p}', '{c} like "{v}"'),
+            TemplatedFilter("like", "like", "{c} like :{p}", '{c} like "{v}"'),
             TemplatedFilter(
-                "notlike", "not like", '"{c}" not like :{p}', '{c} not like "{v}"'
+                "notlike", "not like", "{c} not like :{p}", '{c} not like "{v}"'
             ),
-            TemplatedFilter("glob", "glob", '"{c}" glob :{p}', '{c} glob "{v}"'),
+            TemplatedFilter("glob", "glob", "{c} glob :{p}", '{c} glob "{v}"'),
             InFilter(),
             NotInFilter(),
         ]
@@ -322,13 +366,13 @@ class Filters:
                 TemplatedFilter(
                     "arraycontains",
                     "array contains",
-                    """:{p} in (select value from json_each([{t}].[{c}]))""",
+                    """:{p} in (select value from json_each({t}.{c}))""",
                     '{c} contains "{v}"',
                 ),
                 TemplatedFilter(
                     "arraynotcontains",
                     "array does not contain",
-                    """:{p} not in (select value from json_each([{t}].[{c}]))""",
+                    """:{p} not in (select value from json_each({t}.{c}))""",
                     '{c} does not contain "{v}"',
                 ),
             ]
@@ -336,36 +380,34 @@ class Filters:
             else []
         )
         + [
+            TemplatedFilter("date", "date", "date({c}) = :{p}", '"{c}" is on date {v}'),
             TemplatedFilter(
-                "date", "date", 'date("{c}") = :{p}', '"{c}" is on date {v}'
+                "isnull", "is null", "{c} is null", "{c} is null", no_argument=True
-            ),
-            TemplatedFilter(
-                "isnull", "is null", '"{c}" is null', "{c} is null", no_argument=True
             ),
             TemplatedFilter(
                 "notnull",
                 "is not null",
-                '"{c}" is not null',
+                "{c} is not null",
                 "{c} is not null",
                 no_argument=True,
             ),
             TemplatedFilter(
                 "isblank",
                 "is blank",
-                '("{c}" is null or "{c}" = "")',
+                "({c} is null or {c} = '')",
                 "{c} is blank",
                 no_argument=True,
             ),
             TemplatedFilter(
                 "notblank",
                 "is not blank",
-                '("{c}" is not null and "{c}" != "")',
+                "({c} is not null and {c} != '')",
                 "{c} is not blank",
                 no_argument=True,
             ),
         ]
     )
-    _filters_by_key = {f.key: f for f in _filters}
+    _filters_by_key: ClassVar[dict[str, Filter]] = {f.key: f for f in _filters}
 
     def __init__(self, pairs):
         self.pairs = pairs

datasette/fixtures.py

@@ -0,0 +1,414 @@
+import itertools
+import random
+import string
+
+from datasette.utils import documented
+from datasette.utils.sqlite import sqlite3
+
+__all__ = [
+    "EXTRA_DATABASE_SQL",
+    "TABLES",
+    "TABLE_PARAMETERIZED_SQL",
+    "generate_compound_rows",
+    "generate_sortable_rows",
+    "populate_extra_database",
+    "populate_fixture_database",
+    "write_extra_database",
+    "write_fixture_database",
+]
+
+
+def generate_compound_rows(num):
+    """Generate rows for the compound_three_primary_keys fixture table."""
+    for a, b, c in itertools.islice(
+        itertools.product(string.ascii_lowercase, repeat=3), num
+    ):
+        yield a, b, c, f"{a}-{b}-{c}"
+
+
+def generate_sortable_rows(num):
+    """Generate rows for the sortable fixture table."""
+    rand = random.Random(42)
+    for a, b in itertools.islice(
+        itertools.product(string.ascii_lowercase, repeat=2), num
+    ):
+        yield {
+            "pk1": a,
+            "pk2": b,
+            "content": f"{a}-{b}",
+            "sortable": rand.randint(-100, 100),
+            "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]),
+            "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]),
+            "text": rand.choice(["$null", "$blah"]),
+        }
+
+
+TABLES = (
+    """
+CREATE TABLE simple_primary_key (
+  id integer primary key,
+  content text
+);
+
+CREATE TABLE primary_key_multiple_columns (
+  id varchar(30) primary key,
+  content text,
+  content2 text
+);
+
+CREATE TABLE primary_key_multiple_columns_explicit_label (
+  id varchar(30) primary key,
+  content text,
+  content2 text
+);
+
+CREATE TABLE compound_primary_key (
+  pk1 varchar(30),
+  pk2 varchar(30),
+  content text,
+  PRIMARY KEY (pk1, pk2)
+);
+
+INSERT INTO compound_primary_key VALUES ('a', 'b', 'c');
+INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c');
+INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO');
+
+CREATE TABLE compound_three_primary_keys (
+  pk1 varchar(30),
+  pk2 varchar(30),
+  pk3 varchar(30),
+  content text,
+  PRIMARY KEY (pk1, pk2, pk3)
+);
+CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content);
+
+CREATE TABLE foreign_key_references (
+  pk varchar(30) primary key,
+  foreign_key_with_label integer,
+  foreign_key_with_blank_label integer,
+  foreign_key_with_no_label varchar(30),
+  foreign_key_compound_pk1 varchar(30),
+  foreign_key_compound_pk2 varchar(30),
+  FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id),
+  FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id),
+  FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id)
+  FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2)
+);
+
+CREATE TABLE sortable (
+  pk1 varchar(30),
+  pk2 varchar(30),
+  content text,
+  sortable integer,
+  sortable_with_nulls real,
+  sortable_with_nulls_2 real,
+  text text,
+  PRIMARY KEY (pk1, pk2)
+);
+
+CREATE TABLE no_primary_key (
+  content text,
+  a text,
+  b text,
+  c text
+);
+
+CREATE TABLE [123_starts_with_digits] (
+  content text
+);
+
+CREATE VIEW paginated_view AS
+    SELECT
+        content,
+        '- ' || content || ' -' AS content_extra
+    FROM no_primary_key;
+
+CREATE TABLE "Table With Space In Name" (
+  pk varchar(30) primary key,
+  content text
+);
+
+CREATE TABLE "table/with/slashes.csv" (
+  pk varchar(30) primary key,
+  content text
+);
+
+CREATE TABLE "complex_foreign_keys" (
+  pk varchar(30) primary key,
+  f1 integer,
+  f2 integer,
+  f3 integer,
+  FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id),
+  FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id),
+  FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id)
+);
+
+CREATE TABLE "custom_foreign_key_label" (
+  pk varchar(30) primary key,
+  foreign_key_with_custom_label text,
+  FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id)
+);
+
+CREATE TABLE tags (
+    tag TEXT PRIMARY KEY
+);
+
+CREATE TABLE searchable (
+  pk integer primary key,
+  text1 text,
+  text2 text,
+  [name with . and spaces] text
+);
+
+CREATE TABLE searchable_tags (
+    searchable_id integer,
+    tag text,
+    PRIMARY KEY (searchable_id, tag),
+    FOREIGN KEY (searchable_id) REFERENCES searchable(pk),
+    FOREIGN KEY (tag) REFERENCES tags(tag)
+);
+
+INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther');
+INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma');
+
+INSERT INTO tags VALUES ("canine");
+INSERT INTO tags VALUES ("feline");
+
+INSERT INTO searchable_tags (searchable_id, tag) VALUES
+    (1, "feline"),
+    (2, "canine")
+;
+
+CREATE VIRTUAL TABLE "searchable_fts"
+    USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk");
+INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild');
+
+CREATE TABLE [select] (
+  [group] text,
+  [having] text,
+  [and] text,
+  [json] text
+);
+INSERT INTO [select] VALUES ('group', 'having', 'and',
+    '{"href": "http://example.com/", "label":"Example"}'
+);
+
+CREATE TABLE infinity (
+    value REAL
+);
+INSERT INTO infinity VALUES
+    (1e999),
+    (-1e999),
+    (1.5)
+;
+
+CREATE TABLE facet_cities (
+    id integer primary key,
+    name text
+);
+INSERT INTO facet_cities (id, name) VALUES
+    (1, 'San Francisco'),
+    (2, 'Los Angeles'),
+    (3, 'Detroit'),
+    (4, 'Memnonia')
+;
+
+CREATE TABLE facetable (
+    pk integer primary key,
+    created text,
+    planet_int integer,
+    on_earth integer,
+    state text,
+    _city_id integer,
+    _neighborhood text,
+    tags text,
+    complex_array text,
+    distinct_some_null,
+    n text,
+    FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id)
+);
+INSERT INTO facetable
+    (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n)
+VALUES
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null)
+;
+
+CREATE TABLE binary_data (
+    data BLOB
+);
+
+-- Many 2 Many demo: roadside attractions!
+
+CREATE TABLE roadside_attractions (
+    pk integer primary key,
+    name text,
+    address text,
+    url text,
+    latitude real,
+    longitude real
+);
+INSERT INTO roadside_attractions VALUES (
+    1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/",
+    37.0167, -122.0024
+);
+INSERT INTO roadside_attractions VALUES (
+    2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/",
+    37.3184, -121.9511
+);
+INSERT INTO roadside_attractions VALUES (
+    3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null,
+    37.5793, -122.3442
+);
+INSERT INTO roadside_attractions VALUES (
+    4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/",
+    37.0414, -122.0725
+);
+
+CREATE TABLE attraction_characteristic (
+    pk integer primary key,
+    name text
+);
+INSERT INTO attraction_characteristic VALUES (
+    1, "Museum"
+);
+INSERT INTO attraction_characteristic VALUES (
+    2, "Paranormal"
+);
+
+CREATE TABLE roadside_attraction_characteristics (
+    attraction_id INTEGER REFERENCES roadside_attractions(pk),
+    characteristic_id INTEGER REFERENCES attraction_characteristic(pk)
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    1, 2
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    2, 2
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    4, 2
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    3, 1
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    4, 1
+);
+
+INSERT INTO simple_primary_key VALUES (1, 'hello');
+INSERT INTO simple_primary_key VALUES (2, 'world');
+INSERT INTO simple_primary_key VALUES (3, '');
+INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO');
+INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC');
+
+INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world');
+INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2');
+
+INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b');
+INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null);
+
+INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1);
+INSERT INTO custom_foreign_key_label VALUES (1, 1);
+
+INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey');
+
+CREATE VIEW simple_view AS
+    SELECT content, upper(content) AS upper_content FROM simple_primary_key;
+
+CREATE VIEW searchable_view AS
+    SELECT * from searchable;
+
+CREATE VIEW searchable_view_configured_by_metadata AS
+    SELECT * from searchable;
+
+"""
+    + "\n".join(
+        [
+            'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format(
+                i=i + 1
+            )
+            for i in range(201)
+        ]
+    )
+    + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n'
+    + "\n".join(
+        [
+            f'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'
+            for a, b, c, content in generate_compound_rows(1001)
+        ]
+    )
+    + "\n".join(["""INSERT INTO sortable VALUES (
+        "{pk1}", "{pk2}", "{content}", {sortable},
+        {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}");
+    """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)])
+)
+
+TABLE_PARAMETERIZED_SQL = [
+    ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]),
+    ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]),
+    ("insert into binary_data (data) values (null);", []),
+]
+
+EXTRA_DATABASE_SQL = """
+CREATE TABLE searchable (
+  pk integer primary key,
+  text1 text,
+  text2 text
+);
+
+CREATE VIEW searchable_view AS SELECT * FROM searchable;
+
+INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog');
+INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel');
+
+CREATE VIRTUAL TABLE "searchable_fts"
+    USING FTS3 (text1, text2, content="searchable");
+INSERT INTO "searchable_fts" (rowid, text1, text2)
+    SELECT rowid, text1, text2 FROM searchable;
+"""
+
+
+@documented(label="datasette_fixtures_populate_fixture_database")
+def populate_fixture_database(conn):
+    """Populate a SQLite connection with Datasette's test fixture tables."""
+    conn.executescript(TABLES)
+    for sql, params in TABLE_PARAMETERIZED_SQL:
+        with conn:
+            conn.execute(sql, params)
+
+
+def populate_extra_database(conn):
+    """Populate a SQLite connection with the extra database used in tests."""
+    conn.executescript(EXTRA_DATABASE_SQL)
+
+
+def write_fixture_database(db_filename):
+    """Write Datasette's test fixture tables to a SQLite database file."""
+    conn = sqlite3.connect(db_filename)
+    try:
+        populate_fixture_database(conn)
+    finally:
+        conn.close()
+
+
+def write_extra_database(db_filename):
+    """Write the extra test database tables to a SQLite database file."""
+    conn = sqlite3.connect(db_filename)
+    try:
+        populate_extra_database(conn)
+    finally:
+        conn.close()

datasette/forbidden.py

@@ -1,9 +1,20 @@
-from datasette import hookimpl, Response
+from datasette import Response, hookimpl
+
+from .utils import add_cors_headers
 
 
 @hookimpl(trylast=True)
 def forbidden(datasette, request, message):
     async def inner():
+        if (
+            request.path.split("?")[0].endswith(".json")
+            or "application/json" in (request.headers.get("accept") or "")
+            or request.headers.get("content-type") == "application/json"
+        ):
+            headers = {}
+            if datasette.cors:
+                add_cors_headers(headers)
+            return Response.error(message, 403, headers=headers)
         return Response.html(
             await datasette.render_template(
                 "error.html",

datasette/handle_exception.py

@@ -1,16 +1,21 @@
-from datasette import hookimpl, Response
+import traceback
-from .utils import add_cors_headers
+
+from markupsafe import Markup
+
+from datasette import Response, hookimpl
+
+from .utils import add_cors_headers, error_body
 from .utils.asgi import (
     Base400,
 )
 from .views.base import DatasetteError
-from markupsafe import Markup
-import traceback
 
+# Debugger imports are deliberate - they back the "pdb" setting, which drops
+# into a debugger on unhandled exceptions
 try:
-    import ipdb as pdb
+    import ipdb as pdb  # noqa: T100
 except ImportError:
-    import pdb
+    import pdb  # noqa: T100
 
 try:
     import rich
@@ -28,6 +33,7 @@ def handle_exception(datasette, request, exception):
             rich.get_console().print_exception(show_locals=True)
 
         title = None
+        plain_message = None
         if isinstance(exception, Base400):
             status = exception.status
             info = {}
@@ -36,6 +42,7 @@ def handle_exception(datasette, request, exception):
             status = exception.status
             info = exception.error_dict
             message = exception.message
+            plain_message = exception.plain_message
             if exception.message_is_html:
                 message = Markup(message)
             title = exception.title
@@ -45,6 +52,17 @@ def handle_exception(datasette, request, exception):
             message = str(exception)
             traceback.print_exc()
         templates = [f"{status}.html", "error.html"]
+        headers = {}
+        if datasette.cors:
+            add_cors_headers(headers)
+        if request.path.split("?")[0].endswith(".json"):
+            body = dict(info)
+            body.update(error_body(plain_message or message, status))
+            return Response.json(body, status=status, headers=headers)
+        if request.path.split("?")[0].endswith(".csv"):
+            return Response.text(
+                plain_message or message, status=status, headers=headers
+            )
         info.update(
             {
                 "ok": False,
@@ -53,12 +71,6 @@ def handle_exception(datasette, request, exception):
                 "title": title,
             }
         )
-        headers = {}
-        if datasette.cors:
-            add_cors_headers(headers)
-        if request.path.split("?")[0].endswith(".json"):
-            return Response.json(info, status=status, headers=headers)
-        else:
         environment = datasette.get_jinja_environment(request)
         template = environment.select_template(templates)
         return Response.html(
@@ -66,8 +78,7 @@ def handle_exception(datasette, request, exception):
                 dict(
                     info,
                     urls=datasette.urls,
-                        app_css_hash=datasette.app_css_hash(),
+                    menu_links=list,
-                        menu_links=lambda: [],
                 )
             ),
             status=status,

datasette/hookspecs.py

@@ -1,5 +1,4 @@
-from pluggy import HookimplMarker
+from pluggy import HookimplMarker, HookspecMarker
-from pluggy import HookspecMarker
 
 hookspec = HookspecMarker("datasette")
 hookimpl = HookimplMarker("datasette")
@@ -10,6 +9,11 @@ def startup(datasette):
     """Fires directly after Datasette first starts running"""
 
 
+@hookspec
+def shutdown(datasette):
+    """Called once when the Datasette server is shutting down"""
+
+
 @hookspec
 def asgi_wrapper(datasette):
     """Returns an ASGI middleware callable to wrap our ASGI application with"""
@@ -46,7 +50,7 @@ def extra_body_script(
 def extra_template_vars(
     template, database, table, columns, view_name, request, datasette
 ):
-    """Extra template variables to be made available to the template - can return dict or callable or awaitable"""
+    """Extra template variables to be made available to the template - can return dict, None, callable or awaitable"""
 
 
 @hookspec
@@ -55,7 +59,17 @@ def publish_subcommand(publish):
 
 
 @hookspec
-def render_cell(row, value, column, table, database, datasette, request):
+def render_cell(
+    row,
+    value,
+    column,
+    table,
+    pks,
+    database,
+    datasette,
+    request,
+    column_type,
+):
     """Customize rendering of HTML table cell values"""
 
 
@@ -74,6 +88,11 @@ def register_actions(datasette):
     """Register actions: returns a list of datasette.permission.Action objects"""
 
 
+@hookspec
+def register_column_types(datasette):
+    """Return a list of ColumnType subclasses"""
+
+
 @hookspec
 def register_routes(datasette):
     """Register URL routes: return a list of (regex, view_function) pairs"""
@@ -122,11 +141,6 @@ def permission_resources_sql(datasette, actor, action):
     """
 
 
-@hookspec
-def canned_queries(datasette, database, actor):
-    """Return a dictionary of canned query definitions or an awaitable function that returns them"""
-
-
 @hookspec
 def register_magic_parameters(datasette):
     """Return a list of (name, function) magic parameter functions"""
@@ -142,39 +156,39 @@ def menu_links(datasette, actor, request):
     """Links for the navigation menu"""
 
 
+@hookspec
+def jump_items_sql(datasette, actor, request):
+    """SQL fragments for extra items in the jump menu"""
+
+
 @hookspec
 def row_actions(datasette, actor, request, database, table, row):
-    """Links for the row actions menu"""
+    """Items for the row actions menu"""
 
 
 @hookspec
 def table_actions(datasette, actor, database, table, request):
-    """Links for the table actions menu"""
+    """Items for the table actions menu"""
 
 
 @hookspec
 def view_actions(datasette, actor, database, view, request):
-    """Links for the view actions menu"""
+    """Items for the view actions menu"""
 
 
 @hookspec
 def query_actions(datasette, actor, database, query_name, request, sql, params):
-    """Links for the query and canned query actions menu"""
+    """Items for the query and stored query actions menu"""
 
 
 @hookspec
 def database_actions(datasette, actor, database, request):
-    """Links for the database actions menu"""
+    """Items for the database actions menu"""
 
 
 @hookspec
 def homepage_actions(datasette, actor, request):
-    """Links for the homepage actions menu"""
+    """Items for the homepage actions menu"""
-
-
-@hookspec
-def skip_csrf(datasette, scope):
-    """Mechanism for skipping CSRF checks for certain requests"""
 
 
 @hookspec
@@ -218,5 +232,38 @@ def top_query(datasette, request, database, sql):
 
 
 @hookspec
-def top_canned_query(datasette, request, database, query_name):
+def top_stored_query(datasette, request, database, query_name):
-    """HTML to include at the top of the canned query page"""
+    """HTML to include at the top of the stored query page"""
+
+
+@hookspec
+def register_token_handler(datasette):
+    """Return a TokenHandler instance for token creation and verification"""
+
+
+@hookspec
+def write_wrapper(datasette, database, request, transaction):
+    """Called when a write function is about to execute.
+
+    Return a generator function that accepts a ``conn`` argument and
+    optionally a ``track_event`` argument.  The generator should
+    ``yield`` exactly once: code before the ``yield`` runs before
+    the write, code after the ``yield`` runs after the write
+    completes. The result of the write is sent back through the
+    ``yield``, so you can capture it with ``result = yield``.
+
+    If your generator accepts ``track_event``, you can call
+    ``track_event(event)`` to queue an event that will be dispatched
+    via ``datasette.track_event()`` after the write commits
+    successfully.  Events are discarded if the write raises an
+    exception.
+
+    If the write raises an exception, it is thrown into the generator
+    so you can handle it with a try/except around the ``yield``.
+
+    ``request`` may be ``None`` for writes not originating from an
+    HTTP request.  ``transaction`` is ``True`` if the write will
+    be wrapped in a transaction.
+
+    Return ``None`` to skip wrapping.
+    """

datasette/inspect.py

@@ -1,16 +1,15 @@
 import hashlib
 
 from .utils import (
-    detect_spatialite,
     detect_fts,
     detect_primary_keys,
+    detect_spatialite,
     escape_sqlite,
     get_all_foreign_keys,
-    table_columns,
     sqlite3,
+    table_columns,
 )
 
-
 HASH_BLOCK_SIZE = 1024 * 1024
 
 
@@ -70,16 +69,11 @@ def inspect_tables(conn, database_metadata):
         tables[table]["foreign_keys"] = info
 
     # Mark tables 'hidden' if they relate to FTS virtual tables
-    hidden_tables = [
+    hidden_tables = [r["name"] for r in conn.execute("""
-        r["name"]
-        for r in conn.execute(
-            """
                 select name from sqlite_master
                 where rootpage = 0
                 and sql like '%VIRTUAL TABLE%USING FTS%'
-            """
+            """)]
-        )
-    ]
 
     if detect_spatialite(conn):
         # Also hide Spatialite internal tables
@@ -94,20 +88,17 @@ def inspect_tables(conn, database_metadata):
             "views_geometry_columns",
             "virts_geometry_columns",
         ] + [
-            r["name"]
+            r["name"] for r in conn.execute("""
-            for r in conn.execute(
-                """
                     select name from sqlite_master
                     where name like "idx_%"
                     and type = "table"
-                """
+                """)
-            )
         ]
 
-    for t in tables.keys():
+    for t, table_info in tables.items():
         for hidden_table in hidden_tables:
             if t == hidden_table or t.startswith(hidden_table):
-                tables[t]["hidden"] = True
+                table_info["hidden"] = True
                 continue
 
     return tables

datasette/jump.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class JumpSQL:
+    sql: str
+    params: dict[str, Any] | None = None
+    database: str | None = None
+
+    @classmethod
+    def menu_item(
+        cls,
+        *,
+        label: str,
+        url: str,
+        description: str = "Menu item",
+        search_text: str | None = None,
+        display_name: str | None = None,
+        item_type: str = "menu",
+    ) -> JumpSQL:
+        if search_text is None:
+            search_text = " ".join(
+                text for text in (label, display_name, description) if text is not None
+            )
+        return cls(
+            sql="""
+            SELECT
+                :type AS type,
+                :label AS label,
+                :description AS description,
+                :url AS url,
+                :search_text AS search_text,
+                :display_name AS display_name
+            """,
+            params={
+                "type": item_type,
+                "label": label,
+                "description": description,
+                "url": url,
+                "search_text": search_text,
+                "display_name": display_name,
+            },
+        )
+
+
+_PARAM_RE = re.compile(r"(?<!:):([A-Za-z_][A-Za-z0-9_]*)")
+
+
+def namespace_sql_params(sql: str, params: dict[str, Any], prefix: str):
+    """Rename named SQL parameters so UNION query parameters cannot collide."""
+    if not params:
+        return sql, {}
+
+    renamed = {key: f"{prefix}_{key}" for key in params}
+
+    def replace(match):
+        key = match.group(1)
+        if key not in renamed:
+            return match.group(0)
+        return f":{renamed[key]}"
+
+    return _PARAM_RE.sub(replace, sql), {
+        renamed[key]: value for key, value in params.items()
+    }

datasette/permissions.py

@@ -1,14 +1,25 @@
+import contextvars
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Any, NamedTuple
-import contextvars
 
+_SQLITE_IDENTIFIER_CASE = str.maketrans(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
+)
 
 # Context variable to track when permission checks should be skipped
 _skip_permission_checks = contextvars.ContextVar(
     "skip_permission_checks", default=False
 )
 
+# Request-scoped cache of permission check results. The ASGI router sets
+# this to a fresh dict at the start of each request, so cached verdicts
+# never outlive a request or leak between actors. Keys are
+# (actor_json, action, parent, child) tuples, values are booleans.
+_permission_check_cache: contextvars.ContextVar[dict | None] = contextvars.ContextVar(
+    "permission_check_cache", default=None
+)
+
 
 class SkipPermissions:
     """Context manager to temporarily skip permission checks.
@@ -42,6 +53,15 @@ class Resource(ABC):
     # Class-level metadata (subclasses must define these)
     name: str = None  # e.g., "table", "database", "model"
     parent_class: type["Resource"] | None = None  # e.g., DatabaseResource for tables
+    case_insensitive_child: bool = False
+
+    @classmethod
+    def normalize_child(cls, child: str | None) -> str | None:
+        """Return a comparison key without changing the resource's display name."""
+        if cls.case_insensitive_child and child is not None:
+            # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold.
+            return child.translate(_SQLITE_IDENTIFIER_CASE)
+        return child
 
     # Instance-level optional extra attributes
     reasons: list[str] | None = None
@@ -59,6 +79,16 @@ class Resource(ABC):
         self.child = child
         self._private = None  # Sentinel to track if private was set
 
+    def __str__(self) -> str:
+        return "/".join(
+            str(part) for part in (self.parent, self.child) if part is not None
+        )
+
+    def __repr__(self) -> str:
+        return (
+            f"{self.__class__.__name__}(parent={self.parent!r}, child={self.child!r})"
+        )
+
     @property
     def private(self) -> bool:
         """
@@ -106,13 +136,12 @@ class Resource(ABC):
 
     @classmethod
     @abstractmethod
-    def resources_sql(cls) -> str:
+    async def resources_sql(cls, datasette, actor=None) -> str:
         """
         Return SQL query that returns all resources of this type.
 
         Must return two columns: parent, child
         """
-        pass
 
 
 class AllowedResource(NamedTuple):
@@ -130,6 +159,11 @@ class Action:
     resource_class: type[Resource] | None = None
     also_requires: str | None = None  # Optional action name that must also be allowed
 
+    def normalize_child(self, child: str | None) -> str | None:
+        if self.resource_class is None:
+            return child
+        return self.resource_class.normalize_child(child)
+
     @property
     def takes_parent(self) -> bool:
         """

datasette/plugins.py

@@ -1,19 +1,13 @@
 import importlib
-import os
-import pluggy
-from pprint import pprint
-import sys
-from . import hookspecs
-
-if sys.version_info >= (3, 9):
-    import importlib.resources as importlib_resources
-else:
-    import importlib_resources
-if sys.version_info >= (3, 10):
 import importlib.metadata as importlib_metadata
-else:
+import importlib.resources as importlib_resources
-    import importlib_metadata
+import os
+import sys
+from pprint import pprint
 
+import pluggy
+
+from . import hookspecs
 
 DEFAULT_PLUGINS = (
     "datasette.publish.heroku",
@@ -23,10 +17,17 @@ DEFAULT_PLUGINS = (
     "datasette.sql_functions",
     "datasette.actor_auth_cookie",
     "datasette.default_permissions",
+    "datasette.default_permissions.tokens",
+    "datasette.default_permissions.sqlite_statistics",
     "datasette.default_actions",
+    "datasette.default_column_types",
     "datasette.default_magic_parameters",
     "datasette.blob_renderer",
-    "datasette.default_menu_links",
+    "datasette.default_debug_menu",
+    "datasette.default_jump_items",
+    "datasette.default_database_actions",
+    "datasette.default_table_actions",
+    "datasette.default_query_actions",
     "datasette.handle_exception",
     "datasette.forbidden",
     "datasette.events",
@@ -79,7 +80,7 @@ if DATASETTE_LOAD_PLUGINS is not None:
                     # Ensure name can be found in plugin_to_distinfo later:
                     pm._plugin_distinfo.append((mod, distribution))
         except importlib_metadata.PackageNotFoundError:
-            sys.stderr.write("Plugin {} could not be found\n".format(package_name))
+            sys.stderr.write(f"Plugin {package_name} could not be found\n")
 
 
 # Load default plugins

datasette/publish/cloudrun.py

@@ -1,15 +1,17 @@
-from datasette import hookimpl
-import click
 import json
 import os
 import re
 from subprocess import CalledProcessError, check_call, check_output
 
+import click
+
+from datasette import hookimpl
+
+from ..utils import temporary_docker_directory
 from .common import (
     add_common_publish_arguments_and_options,
     fail_if_publish_binary_not_installed,
 )
-from ..utils import temporary_docker_directory
 
 
 @hookimpl
@@ -219,7 +221,7 @@ def publish_subcommand(publish):
 
             check_call(
                 "gcloud builds submit --tag {}{}".format(
-                    image_id, " --timeout {}".format(timeout) if timeout else ""
+                    image_id, f" --timeout {timeout}" if timeout else ""
                 ),
                 shell=True,
             )
@@ -231,7 +233,7 @@ def publish_subcommand(publish):
             ("--min-instances", min_instances),
         ):
             if value is not None:
-                extra_deploy_options.append("{} {}".format(option, value))
+                extra_deploy_options.append(f"{option} {value}")
         check_call(
             "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format(
                 image_id,
@@ -258,24 +260,16 @@ def _ensure_artifact_registry(artifact_project, artifact_region, artifact_reposi
         ) from exc
 
     describe_cmd = (
-        "gcloud artifacts repositories describe {repo} --project {project} "
+        f"gcloud artifacts repositories describe {artifact_repository} --project {artifact_project} "
-        "--location {location} --quiet"
+        f"--location {artifact_region} --quiet"
-    ).format(
-        repo=artifact_repository,
-        project=artifact_project,
-        location=artifact_region,
     )
     try:
         check_call(describe_cmd, shell=True)
         return
     except CalledProcessError:
         create_cmd = (
-            "gcloud artifacts repositories create {repo} --repository-format=docker "
+            f"gcloud artifacts repositories create {artifact_repository} --repository-format=docker "
-            '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet'
+            f'--location {artifact_region} --project {artifact_project} --description "Datasette Cloud Run images" --quiet'
-        ).format(
-            repo=artifact_repository,
-            location=artifact_region,
-            project=artifact_project,
         )
         try:
             check_call(create_cmd, shell=True)

datasette/publish/common.py

@@ -1,9 +1,11 @@
-from ..utils import StaticMount
-import click
 import os
 import shutil
 import sys
 
+import click
+
+from ..utils import StaticMount
+
 
 def add_common_publish_arguments_and_options(subcommand):
     for decorator in reversed(
@@ -76,9 +78,7 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link):
     """Exit (with error message) if ``binary` isn't installed"""
     if not shutil.which(binary):
         click.secho(
-            "Publishing to {publish_target} requires {binary} to be installed and configured".format(
+            f"Publishing to {publish_target} requires {binary} to be installed and configured",
-                publish_target=publish_target, binary=binary
-            ),
             bg="red",
             fg="white",
             bold=True,

datasette/publish/heroku.py

@@ -1,19 +1,21 @@
-from contextlib import contextmanager
-from datasette import hookimpl
-import click
 import json
 import os
 import pathlib
 import shlex
 import shutil
-from subprocess import call, check_output
 import tempfile
+from contextlib import contextmanager
+from subprocess import call, check_output
+
+import click
+
+from datasette import hookimpl
+from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata
 
 from .common import (
     add_common_publish_arguments_and_options,
     fail_if_publish_binary_not_installed,
 )
-from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata
 
 
 @hookimpl
@@ -234,7 +236,7 @@ def temporary_heroku_directory(
             extras.extend(["--static", f"{mount_point}:{mount_point}"])
 
         quoted_files = " ".join(
-            ["-i {}".format(shlex.quote(file_name)) for file_name in file_names]
+            [f"-i {shlex.quote(file_name)}" for file_name in file_names]
         )
         procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format(
             quoted_files=quoted_files, extras=" ".join(extras)

datasette/renderer.py

@@ -1,10 +1,13 @@
 import json
+
+from datasette.extras import extra_names_from_request
 from datasette.utils import (
-    value_as_boolean,
-    remove_infinites,
     CustomJSONEncoder,
+    error_body,
     path_from_row_pks,
+    remove_infinites,
     sqlite3,
+    value_as_boolean,
 )
 from datasette.utils.asgi import Response
 
@@ -51,8 +54,7 @@ def json_renderer(request, args, data, error, truncated=None):
     if error:
         shape = "objects"
         status_code = 400
-        data["error"] = error
+        data.update(error_body(error, status_code))
-        data["ok"] = False
 
     if truncated is not None:
         data["truncated"] = truncated
@@ -86,7 +88,8 @@ def json_renderer(request, args, data, error, truncated=None):
                         object_rows[pk_string] = row
                     data = object_rows
             if shape_error:
-                data = {"ok": False, "error": shape_error}
+                status_code = 400
+                data = error_body(shape_error, status_code)
         elif shape == "array":
             data = data["rows"]
 
@@ -99,16 +102,11 @@ def json_renderer(request, args, data, error, truncated=None):
             data["rows"] = [list(row.values()) for row in data["rows"]]
     else:
         status_code = 400
-        data = {
+        data = error_body(f"Invalid _shape: {shape}", status_code)
-            "ok": False,
-            "error": f"Invalid _shape: {shape}",
-            "status": 400,
-            "title": None,
-        }
 
     # Don't include "columns" in output
     # https://github.com/simonw/datasette/issues/2136
-    if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"):
+    if isinstance(data, dict) and "columns" not in extra_names_from_request(request):
         data.pop("columns", None)
 
     # Handle _nl option for _shape=array

datasette/resources.py

@@ -13,7 +13,7 @@ class DatabaseResource(Resource):
         super().__init__(parent=database, child=None)
 
     @classmethod
-    async def resources_sql(cls, datasette) -> str:
+    async def resources_sql(cls, datasette, actor=None) -> str:
         return """
             SELECT database_name AS parent, NULL AS child
             FROM catalog_databases
@@ -25,12 +25,13 @@ class TableResource(Resource):
 
     name = "table"
     parent_class = DatabaseResource
+    case_insensitive_child = True
 
     def __init__(self, database: str, table: str):
         super().__init__(parent=database, child=table)
 
     @classmethod
-    async def resources_sql(cls, datasette) -> str:
+    async def resources_sql(cls, datasette, actor=None) -> str:
         return """
             SELECT database_name AS parent, table_name AS child
             FROM catalog_tables
@@ -41,7 +42,7 @@ class TableResource(Resource):
 
 
 class QueryResource(Resource):
-    """A canned query in a database."""
+    """A stored query in a database."""
 
     name = "query"
     parent_class = DatabaseResource
@@ -50,41 +51,9 @@ class QueryResource(Resource):
         super().__init__(parent=database, child=query)
 
     @classmethod
-    async def resources_sql(cls, datasette) -> str:
+    async def resources_sql(cls, datasette, actor=None) -> str:
-        from datasette.plugins import pm
+        return """
-        from datasette.utils import await_me_maybe
+            SELECT q.database_name AS parent, q.name AS child
-
+            FROM queries q
-        # Get all databases from catalog
+            JOIN catalog_databases cd ON cd.database_name = q.database_name
-        db = datasette.get_internal_database()
+        """
-        result = await db.execute("SELECT database_name FROM catalog_databases")
-        databases = [row[0] for row in result.rows]
-
-        # Gather all canned queries from all databases
-        query_pairs = []
-        for database_name in databases:
-            # Call the hook to get queries (including from config via default plugin)
-            for queries_result in pm.hook.canned_queries(
-                datasette=datasette,
-                database=database_name,
-                actor=None,  # Get ALL queries for resource enumeration
-            ):
-                queries = await await_me_maybe(queries_result)
-                if queries:
-                    for query_name in queries.keys():
-                        query_pairs.append((database_name, query_name))
-
-        # Build SQL
-        if not query_pairs:
-            return "SELECT NULL AS parent, NULL AS child WHERE 0"
-
-        # Generate UNION ALL query
-        selects = []
-        for db_name, query_name in query_pairs:
-            # Escape single quotes by doubling them
-            db_escaped = db_name.replace("'", "''")
-            query_escaped = query_name.replace("'", "''")
-            selects.append(
-                f"SELECT '{db_escaped}' AS parent, '{query_escaped}' AS child"
-            )
-
-        return " UNION ALL ".join(selects)

datasette/static/autocomplete.js

@@ -0,0 +1,344 @@
+(function () {
+  function autocompleteValueFromRow(row) {
+    var pks = (row && row.pks) || {};
+    var keys = Object.keys(pks);
+    if (!keys.length) {
+      return "";
+    }
+    if (keys.length === 1) {
+      return String(pks[keys[0]]);
+    }
+    return keys
+      .map(function (key) {
+        return key + "=" + pks[key];
+      })
+      .join(", ");
+  }
+
+  function autocompleteLabelFromRow(row) {
+    var value = autocompleteValueFromRow(row);
+    if (row.label && String(row.label) !== value) {
+      return row.label + " (" + value + ")";
+    }
+    return value;
+  }
+
+  if (!window.customElements || customElements.get("datasette-autocomplete")) {
+    return;
+  }
+
+  class DatasetteAutocomplete extends HTMLElement {
+    constructor() {
+      super();
+      this.input = null;
+      this.listbox = null;
+      this.status = null;
+      this.results = [];
+      this.activeIndex = -1;
+      this.fetchId = 0;
+      this.searchTimer = null;
+      this.boundInput = this.handleInput.bind(this);
+      this.boundKeydown = this.handleKeydown.bind(this);
+      this.boundBlur = this.handleBlur.bind(this);
+      this.boundFocus = this.handleFocus.bind(this);
+      this.boundPositionListbox = this.positionListbox.bind(this);
+    }
+
+    connectedCallback() {
+      if (this.input) {
+        return;
+      }
+      this.input = this.querySelector("input");
+      if (!this.input) {
+        return;
+      }
+
+      var inputId =
+        this.input.id ||
+        "datasette-autocomplete-" + Math.random().toString(36).slice(2);
+      this.input.id = inputId;
+      var listboxId = inputId + "-listbox";
+      var statusId = inputId + "-status";
+
+      this.classList.add("datasette-autocomplete");
+      this.input.setAttribute("role", "combobox");
+      this.input.setAttribute("aria-autocomplete", "list");
+      this.input.setAttribute("aria-expanded", "false");
+      this.input.setAttribute("aria-controls", listboxId);
+      this.input.setAttribute("autocomplete", "off");
+
+      this.listbox = document.createElement("div");
+      this.listbox.className = "datasette-autocomplete-list";
+      this.listbox.id = listboxId;
+      this.listbox.setAttribute("role", "listbox");
+      this.listbox.hidden = true;
+
+      this.status = document.createElement("span");
+      this.status.className = "datasette-autocomplete-status";
+      this.status.id = statusId;
+      this.status.setAttribute("role", "status");
+      this.status.setAttribute("aria-live", "polite");
+
+      this.input.setAttribute(
+        "aria-describedby",
+        [this.input.getAttribute("aria-describedby"), statusId]
+          .filter(Boolean)
+          .join(" "),
+      );
+
+      this.appendChild(this.listbox);
+      this.appendChild(this.status);
+
+      this.input.addEventListener("input", this.boundInput);
+      this.input.addEventListener("keydown", this.boundKeydown);
+      this.input.addEventListener("blur", this.boundBlur);
+      this.input.addEventListener("focus", this.boundFocus);
+    }
+
+    disconnectedCallback() {
+      if (!this.input) {
+        return;
+      }
+      this.input.removeEventListener("input", this.boundInput);
+      this.input.removeEventListener("keydown", this.boundKeydown);
+      this.input.removeEventListener("blur", this.boundBlur);
+      this.input.removeEventListener("focus", this.boundFocus);
+    }
+
+    handleInput() {
+      this.scheduleSearch();
+    }
+
+    handleFocus() {
+      if (this.input.value.trim() || this.hasAttribute("suggest-on-focus")) {
+        this.scheduleSearch();
+      }
+    }
+
+    handleBlur() {
+      window.setTimeout(() => this.close(), 150);
+    }
+
+    handleKeydown(ev) {
+      if (ev.key === "Escape") {
+        if (!this.listbox.hidden) {
+          ev.preventDefault();
+          this.close();
+        }
+        return;
+      }
+      if (ev.key === "ArrowDown") {
+        ev.preventDefault();
+        if (this.listbox.hidden) {
+          this.scheduleSearch();
+        } else {
+          this.setActiveIndex(this.activeIndex + 1);
+        }
+        return;
+      }
+      if (ev.key === "ArrowUp") {
+        ev.preventDefault();
+        if (!this.listbox.hidden) {
+          this.setActiveIndex(this.activeIndex - 1);
+        }
+        return;
+      }
+      if (ev.key === "Enter" && !this.listbox.hidden && this.activeIndex >= 0) {
+        ev.preventDefault();
+        this.chooseIndex(this.activeIndex);
+      }
+    }
+
+    scheduleSearch() {
+      window.clearTimeout(this.searchTimer);
+      this.searchTimer = window.setTimeout(() => this.search(), 150);
+    }
+
+    async search() {
+      var query = this.input.value.trim();
+      var initial = !query && this.hasAttribute("suggest-on-focus");
+      if (!query && !initial) {
+        this.close();
+        this.status.textContent = "";
+        return;
+      }
+      var src = this.getAttribute("src");
+      if (!src) {
+        return;
+      }
+
+      var url = new URL(src, location.href);
+      url.searchParams.set("q", query);
+      if (initial) {
+        url.searchParams.set("_initial", "1");
+      } else {
+        url.searchParams.delete("_initial");
+      }
+      var fetchId = this.fetchId + 1;
+      this.fetchId = fetchId;
+      this.status.textContent = "Searching...";
+
+      try {
+        var response = await fetch(url.toString(), {
+          headers: {
+            Accept: "application/json",
+          },
+        });
+        if (!response.ok) {
+          throw new Error("HTTP " + response.status);
+        }
+        var data = await response.json();
+        if (fetchId !== this.fetchId) {
+          return;
+        }
+        this.results = (data && data.rows) || [];
+        this.render();
+      } catch (_error) {
+        if (fetchId !== this.fetchId) {
+          return;
+        }
+        this.results = [];
+        this.close();
+        this.status.textContent = "Could not load suggestions";
+      }
+    }
+
+    render() {
+      this.listbox.textContent = "";
+      this.activeIndex = -1;
+      if (!this.results.length) {
+        this.close();
+        this.status.textContent = "No matches";
+        return;
+      }
+
+      this.results.forEach((row, index) => {
+        var option = document.createElement("div");
+        option.className = "datasette-autocomplete-option";
+        option.id = this.input.id + "-option-" + index;
+        option.setAttribute("role", "option");
+        option.setAttribute("aria-selected", "false");
+        option.dataset.index = String(index);
+        option.dataset.value = autocompleteValueFromRow(row);
+        option.textContent = autocompleteLabelFromRow(row);
+        option.addEventListener("mousedown", (ev) => {
+          ev.preventDefault();
+          this.chooseIndex(index);
+        });
+        this.listbox.appendChild(option);
+      });
+
+      this.listbox.hidden = false;
+      this.input.setAttribute("aria-expanded", "true");
+      this.status.textContent =
+        this.results.length + (this.results.length === 1 ? " match" : " matches");
+      this.positionListbox();
+      this.setActiveIndex(0);
+    }
+
+    positionListbox() {
+      if (!this.input || !this.listbox || this.listbox.hidden) {
+        return;
+      }
+
+      var gap = 3;
+      var margin = 8;
+      var inputRect = this.input.getBoundingClientRect();
+      this.listbox.style.maxHeight = "";
+      var defaultMaxHeight = parseFloat(
+        window.getComputedStyle(this.listbox).maxHeight,
+      );
+      if (!Number.isFinite(defaultMaxHeight)) {
+        defaultMaxHeight = 256;
+      }
+      var scrollHeight = Math.ceil(this.listbox.scrollHeight);
+      var desiredHeight = Math.min(scrollHeight, defaultMaxHeight);
+      var availableBelow = Math.max(
+        0,
+        (window.innerHeight || document.documentElement.clientHeight) -
+          inputRect.bottom -
+          gap -
+          margin,
+      );
+
+      this.listbox.style.left = inputRect.left + "px";
+      this.listbox.style.top = inputRect.bottom + gap + "px";
+      this.listbox.style.width = inputRect.width + "px";
+      if (scrollHeight <= defaultMaxHeight && scrollHeight <= availableBelow) {
+        this.listbox.style.maxHeight = "none";
+      } else {
+        this.listbox.style.maxHeight =
+          Math.min(defaultMaxHeight, desiredHeight, availableBelow || defaultMaxHeight) +
+          "px";
+      }
+      window.addEventListener("resize", this.boundPositionListbox);
+      document.addEventListener("scroll", this.boundPositionListbox, true);
+    }
+
+    setActiveIndex(index) {
+      var options = this.listbox.querySelectorAll("[role='option']");
+      if (!options.length) {
+        this.activeIndex = -1;
+        this.input.removeAttribute("aria-activedescendant");
+        return;
+      }
+      if (index < 0) {
+        index = options.length - 1;
+      }
+      if (index >= options.length) {
+        index = 0;
+      }
+      options.forEach((option, optionIndex) => {
+        option.setAttribute(
+          "aria-selected",
+          optionIndex === index ? "true" : "false",
+        );
+      });
+      this.activeIndex = index;
+      this.input.setAttribute("aria-activedescendant", options[index].id);
+    }
+
+    chooseIndex(index) {
+      var row = this.results[index];
+      if (!row) {
+        return;
+      }
+      var value = autocompleteValueFromRow(row);
+      var label = autocompleteLabelFromRow(row);
+      this.input.value = value;
+      this.input.dispatchEvent(new Event("change", { bubbles: true }));
+      this.close();
+      this.status.textContent = "Selected " + label;
+      this.dispatchEvent(
+        new CustomEvent("datasette-autocomplete-select", {
+          bubbles: true,
+          detail: {
+            row: row,
+            value: value,
+            label: label,
+          },
+        }),
+      );
+    }
+
+    close() {
+      if (this.listbox) {
+        this.listbox.hidden = true;
+        this.listbox.textContent = "";
+        this.listbox.style.left = "";
+        this.listbox.style.maxHeight = "";
+        this.listbox.style.top = "";
+        this.listbox.style.width = "";
+      }
+      if (this.input) {
+        this.input.setAttribute("aria-expanded", "false");
+        this.input.removeAttribute("aria-activedescendant");
+      }
+      window.removeEventListener("resize", this.boundPositionListbox);
+      document.removeEventListener("scroll", this.boundPositionListbox, true);
+      this.activeIndex = -1;
+    }
+  }
+
+  customElements.define("datasette-autocomplete", DatasetteAutocomplete);
+})();

datasette/static/column-chooser.js

@@ -0,0 +1,380 @@
+let columnChooserInstanceCounter = 0;
+
+class ColumnChooser extends HTMLElement {
+  constructor() {
+    super();
+    this.titleId = `column-chooser-title-${++columnChooserInstanceCounter}`;
+
+    // State
+    this._items = [];
+    this._checked = new Set();
+    this._savedItems = null;
+    this._savedChecked = null;
+    this._onApply = null;
+
+    // Drag state
+    this._ghost = null;
+    this._dragSrcIdx = null;
+    this._dropTargetIdx = null;
+    this._dropPosition = null;
+    this._ghostOffX = 0;
+    this._ghostOffY = 0;
+    this._autoScrollRAF = null;
+    this._lastPointerY = 0;
+    this._lastPointerX = 0;
+    this._SCROLL_ZONE = 72;
+    this._SCROLL_SPEED = 0.4;
+
+    // Bound handlers
+    this._onMove = this._onMove.bind(this);
+    this._onUp = this._onUp.bind(this);
+  }
+
+  connectedCallback() {
+    if (this._modal) return;
+    this.innerHTML = `
+      <datasette-modal><dialog aria-labelledby="${this.titleId}">
+          <div class="modal-header">
+            <span class="modal-title" id="${this.titleId}">Choose columns</span>
+            <span class="modal-meta"></span>
+          </div>
+          <div class="list-toolbar">
+            <button class="select-all">Select all</button>
+            <button class="deselect-all">Deselect all</button>
+          </div>
+          <div class="modal-body list-wrap">
+            <div class="scroll-pulse top"></div>
+            <div class="scroll-pulse bot"></div>
+            <ul class="drag-list"></ul>
+          </div>
+          <div class="modal-footer">
+            <span class="footer-info"></span>
+            <button class="modal-btn modal-btn-ghost">Cancel</button>
+            <button class="modal-btn modal-btn-primary">Apply</button>
+          </div>
+      </dialog></datasette-modal>
+    `;
+
+    // DOM refs
+    this._modal = this.querySelector("datasette-modal");
+    this._listWrap = this.querySelector(".list-wrap");
+    this._dragList = this.querySelector(".drag-list");
+    this._pulseTop = this.querySelector(".scroll-pulse.top");
+    this._pulseBot = this.querySelector(".scroll-pulse.bot");
+    this._selectAllBtn = this.querySelector(".select-all");
+    this._deselectAllBtn = this.querySelector(".deselect-all");
+    this._cancelBtn = this.querySelector(".modal-btn-ghost");
+    this._applyBtn = this.querySelector(".modal-btn-primary");
+    this._countEl = this.querySelector(".modal-meta");
+    this._footerEl = this.querySelector(".footer-info");
+
+    // Event listeners
+    this._selectAllBtn.addEventListener("click", () => this._selectAll());
+    this._deselectAllBtn.addEventListener("click", () => this._deselectAll());
+    this._cancelBtn.addEventListener("click", () =>
+      this._modal.requestClose("cancel"),
+    );
+    this._applyBtn.addEventListener("click", () => this._apply());
+    this._modal.beforeClose = () => {
+      this._items = this._savedItems ? [...this._savedItems] : this._items;
+      this._checked = this._savedChecked
+        ? new Set(this._savedChecked)
+        : this._checked;
+      return true;
+    };
+  }
+
+  /**
+   * Open the column chooser dialog.
+   * @param {Object} opts
+   * @param {string[]} opts.columns - All available column names, in display order.
+   * @param {string[]} opts.selected - Column names that should be pre-checked.
+   * @param {function(string[]): void} opts.onApply - Called with the selected columns in order when Apply is clicked.
+   */
+  open({ columns, selected = [], onApply }) {
+    this._items = [...columns];
+    this._checked = new Set(selected);
+    this._onApply = onApply || null;
+
+    // Save state for cancel/restore
+    this._savedItems = [...this._items];
+    this._savedChecked = new Set(this._checked);
+
+    this._render();
+    this._modal.show();
+  }
+
+  // ── Internal methods ──
+
+  _selectAll() {
+    this._items.forEach((col) => this._checked.add(col));
+    this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => {
+      cb.checked = true;
+    });
+    this._updateCounts();
+  }
+
+  _deselectAll() {
+    this._checked.clear();
+    this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => {
+      cb.checked = false;
+    });
+    this._updateCounts();
+  }
+
+  _apply() {
+    const selected = this._items.filter((col) => this._checked.has(col));
+    this._modal.close();
+    if (this._onApply) {
+      this._onApply(selected);
+    }
+  }
+
+  _render() {
+    this._dragList.innerHTML = "";
+    this._items.forEach((col, i) => {
+      const li = document.createElement("li");
+      li.className = "drag-item";
+      li.dataset.idx = i;
+      li.innerHTML = `
+        <span class="drag-handle" aria-label="Drag to reorder">
+          <svg width="12" height="18" viewBox="0 0 12 18" fill="currentColor">
+            <circle cx="3.5" cy="3.5" r="1.8"/>
+            <circle cx="8.5" cy="3.5" r="1.8"/>
+            <circle cx="3.5" cy="9" r="1.8"/>
+            <circle cx="8.5" cy="9" r="1.8"/>
+            <circle cx="3.5" cy="14.5" r="1.8"/>
+            <circle cx="8.5" cy="14.5" r="1.8"/>
+          </svg>
+        </span>
+        <label class="drag-item-content">
+          <span class="drag-item-check">
+            <input type="checkbox" ${this._checked.has(col) ? "checked" : ""}>
+          </span>
+          <span class="drag-item-label"></span>
+        </label>
+        <div class="drop-indicator"></div>
+      `;
+
+      li.querySelector(".drag-item-label").textContent = col;
+
+      li.querySelector("input").addEventListener("change", (e) => {
+        e.target.checked ? this._checked.add(col) : this._checked.delete(col);
+        this._updateCounts();
+      });
+
+      li.querySelector(".drag-handle").addEventListener("pointerdown", (e) =>
+        this._startDrag(e, i),
+      );
+      this._dragList.appendChild(li);
+    });
+
+    this._updateCounts();
+  }
+
+  _updateCounts() {
+    const n = this._checked.size;
+    this._countEl.textContent = `${n} of ${this._items.length} selected`;
+    this._footerEl.textContent = `${this._items.length} columns`;
+  }
+
+  // ── Drag engine ──
+
+  _startDrag(e, idx) {
+    e.preventDefault();
+    this._dragSrcIdx = idx;
+
+    const srcEl = this._dragList.children[idx];
+    const rect = srcEl.getBoundingClientRect();
+
+    this._ghostOffX = e.clientX - rect.left;
+    this._ghostOffY = e.clientY - rect.top;
+
+    // Keep the drag preview inside the dialog so it stays above the backdrop.
+    this._ghost = document.createElement("div");
+    this._ghost.className = "drag-ghost";
+    this._ghost.style.width = rect.width + "px";
+    this._ghost.style.height = rect.height + "px";
+    this._ghost.innerHTML = srcEl.innerHTML;
+    this._ghost.querySelector(".drop-indicator")?.remove();
+    const h = this._ghost.querySelector(".drag-handle");
+    if (h) h.style.color = "var(--accent)";
+    this._modal.dialog.appendChild(this._ghost);
+
+    srcEl.classList.add("is-dragging");
+    this._positionGhost(e.clientX, e.clientY);
+
+    document.addEventListener("pointermove", this._onMove);
+    document.addEventListener("pointerup", this._onUp);
+    document.addEventListener("pointercancel", this._onUp);
+  }
+
+  _positionGhost(cx, cy) {
+    this._ghost.style.left = cx - this._ghostOffX + "px";
+    this._ghost.style.top = cy - this._ghostOffY + "px";
+  }
+
+  _onMove(e) {
+    this._lastPointerX = e.clientX;
+    this._lastPointerY = e.clientY;
+    this._positionGhost(e.clientX, e.clientY);
+    this._updateDropTarget(e.clientY);
+    this._updateAutoScroll(e.clientY);
+  }
+
+  _onUp() {
+    document.removeEventListener("pointermove", this._onMove);
+    document.removeEventListener("pointerup", this._onUp);
+    document.removeEventListener("pointercancel", this._onUp);
+
+    this._stopAutoScroll();
+
+    const noMove =
+      this._dropTargetIdx === null || this._dropTargetIdx === this._dragSrcIdx;
+    this._clearDropIndicators();
+
+    let dest = null;
+    if (!noMove) {
+      const moved = this._items.splice(this._dragSrcIdx, 1)[0];
+      dest = this._dropTargetIdx;
+      if (this._dropPosition === "after") dest++;
+      if (dest > this._dragSrcIdx) dest--;
+      this._items.splice(dest, 0, moved);
+    }
+
+    this._dragSrcIdx = null;
+    this._dropTargetIdx = null;
+    this._dropPosition = null;
+
+    const g = this._ghost;
+    this._ghost = null;
+
+    if (noMove) {
+      if (g) g.remove();
+      this._render();
+      return;
+    }
+
+    this._render();
+
+    if (g && dest !== null) {
+      const landedEl = this._dragList.children[dest];
+      if (landedEl) {
+        landedEl.style.opacity = "0";
+        const r = landedEl.getBoundingClientRect();
+        g.getBoundingClientRect();
+        g.style.transition =
+          "left 0.15s cubic-bezier(0.22, 1, 0.36, 1), top 0.15s cubic-bezier(0.22, 1, 0.36, 1), box-shadow 0.15s, opacity 0.1s 0.1s";
+        g.style.left = r.left + "px";
+        g.style.top = r.top + "px";
+        g.style.boxShadow = "0 1px 4px rgba(0,0,0,0.08)";
+        g.style.opacity = "0";
+        setTimeout(() => {
+          g.remove();
+          if (landedEl) landedEl.style.opacity = "";
+        }, 160);
+      } else {
+        g.remove();
+      }
+    } else if (g) {
+      g.remove();
+    }
+  }
+
+  _updateDropTarget(clientY) {
+    this._clearDropIndicators();
+    const listItems = [
+      ...this._dragList.querySelectorAll(".drag-item:not(.is-dragging)"),
+    ];
+    if (!listItems.length) return;
+
+    let best = null,
+      bestDist = Infinity;
+    listItems.forEach((li) => {
+      const r = li.getBoundingClientRect();
+      const mid = r.top + r.height / 2;
+      const dist = Math.abs(clientY - mid);
+      if (dist < bestDist) {
+        bestDist = dist;
+        best = li;
+      }
+    });
+
+    if (!best) return;
+    const r = best.getBoundingClientRect();
+    const mid = r.top + r.height / 2;
+    const above = clientY < mid;
+    const indic = best.querySelector(".drop-indicator");
+
+    this._dropTargetIdx = parseInt(best.dataset.idx);
+    this._dropPosition = above ? "before" : "after";
+
+    if (indic) {
+      indic.className = "drop-indicator " + (above ? "top" : "bottom");
+    }
+  }
+
+  _clearDropIndicators() {
+    this._dragList.querySelectorAll(".drop-indicator").forEach((el) => {
+      el.className = "drop-indicator";
+    });
+  }
+
+  _updateAutoScroll(clientY) {
+    const rect = this._listWrap.getBoundingClientRect();
+    const relY = clientY - rect.top;
+    const distTop = relY;
+    const distBot = rect.height - relY;
+
+    const inTop = distTop < this._SCROLL_ZONE && distTop >= 0;
+    const inBot = distBot < this._SCROLL_ZONE && distBot >= 0;
+
+    this._pulseTop.classList.toggle("active", inTop);
+    this._pulseBot.classList.toggle("active", inBot);
+
+    if ((inTop || inBot) && !this._autoScrollRAF) {
+      let lastTime = null;
+      const loop = (ts) => {
+        if (!this._ghost) {
+          this._stopAutoScroll();
+          return;
+        }
+        if (lastTime !== null) {
+          const dt = ts - lastTime;
+          const rect2 = this._listWrap.getBoundingClientRect();
+          const relY2 = this._lastPointerY - rect2.top;
+          const dTop = relY2;
+          const dBot = rect2.height - relY2;
+
+          if (dTop < this._SCROLL_ZONE && dTop >= 0) {
+            const factor = 1 - dTop / this._SCROLL_ZONE;
+            this._listWrap.scrollTop -= this._SCROLL_SPEED * dt * factor * 2.5;
+          } else if (dBot < this._SCROLL_ZONE && dBot >= 0) {
+            const factor = 1 - dBot / this._SCROLL_ZONE;
+            this._listWrap.scrollTop += this._SCROLL_SPEED * dt * factor * 2.5;
+          } else {
+            this._stopAutoScroll();
+            return;
+          }
+          this._updateDropTarget(this._lastPointerY);
+        }
+        lastTime = ts;
+        this._autoScrollRAF = requestAnimationFrame(loop);
+      };
+      this._autoScrollRAF = requestAnimationFrame(loop);
+    }
+
+    if (!inTop && !inBot) this._stopAutoScroll();
+  }
+
+  _stopAutoScroll() {
+    if (this._autoScrollRAF) {
+      cancelAnimationFrame(this._autoScrollRAF);
+      this._autoScrollRAF = null;
+    }
+    this._pulseTop.classList.remove("active");
+    this._pulseBot.classList.remove("active");
+  }
+}
+
+customElements.define("column-chooser", ColumnChooser);

datasette/static/datasette-manager.js

@@ -82,6 +82,48 @@ const datasetteManager = {
     return columnActions;
   },
 
+  /**
+   * Allows JavaScript plugins to replace or enhance insert/edit modal fields
+   * for specific Datasette column types.
+   *
+   * The first plugin to return a control object wins. Returning null or
+   * undefined means "I do not handle this field".
+   */
+  makeColumnField: (context) => {
+    for (const [pluginName, plugin] of datasetteManager.plugins) {
+      if (!plugin.makeColumnField) {
+        continue;
+      }
+      let control = null;
+      try {
+        control = plugin.makeColumnField(context);
+      } catch (error) {
+        console.error(
+          `Error in makeColumnField() for plugin ${pluginName}`,
+          error,
+        );
+        continue;
+      }
+      if (control) {
+        return Object.assign({ pluginName }, control);
+      }
+    }
+    return null;
+  },
+
+  makeJumpSections: (context) => {
+    let jumpSections = [];
+
+    datasetteManager.plugins.forEach((plugin) => {
+      if (plugin.makeJumpSections) {
+        const sections = plugin.makeJumpSections(context) || [];
+        jumpSections.push(...sections);
+      }
+    });
+
+    return jumpSections;
+  },
+
   /**
    * In MVP, each plugin can only have 1 instance.
    * In future, panels could be repeated. We omit that for now since so many plugins depend on
@@ -192,7 +234,6 @@ const initializeDatasette = () => {
   // DATASETTE_EVENTS.INIT event to avoid the habit of reading from the window.
 
   window.__DATASETTE__ = datasetteManager;
-  console.debug("Datasette Manager Created!");
 
   const initDatasetteEvent = new CustomEvent(DATASETTE_EVENTS.INIT, {
     detail: datasetteManager,

datasette/static/json-format-highlight-1.0.1.js

@@ -1,56 +0,0 @@
-/*
-https://github.com/luyilin/json-format-highlight
-From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js
-MIT Licensed
-*/
-(function (global, factory) {
-  typeof exports === "object" && typeof module !== "undefined"
-    ? (module.exports = factory())
-    : typeof define === "function" && define.amd
-      ? define(factory)
-      : (global.jsonFormatHighlight = factory());
-})(this, function () {
-  "use strict";
-
-  var defaultColors = {
-    keyColor: "dimgray",
-    numberColor: "lightskyblue",
-    stringColor: "lightcoral",
-    trueColor: "lightseagreen",
-    falseColor: "#f66578",
-    nullColor: "cornflowerblue",
-  };
-
-  function index(json, colorOptions) {
-    if (colorOptions === void 0) colorOptions = {};
-
-    if (!json) {
-      return;
-    }
-    if (typeof json !== "string") {
-      json = JSON.stringify(json, null, 2);
-    }
-    var colors = Object.assign({}, defaultColors, colorOptions);
-    json = json.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
-    return json.replace(
-      /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g,
-      function (match) {
-        var color = colors.numberColor;
-        if (/^"/.test(match)) {
-          color = /:$/.test(match) ? colors.keyColor : colors.stringColor;
-        } else {
-          color = /true/.test(match)
-            ? colors.trueColor
-            : /false/.test(match)
-              ? colors.falseColor
-              : /null/.test(match)
-                ? colors.nullColor
-                : color;
-        }
-        return '<span style="color: ' + color + '">' + match + "</span>";
-      },
-    );
-  }
-
-  return index;
-});

datasette/static/mobile-column-actions.js

@@ -0,0 +1,294 @@
+var MOBILE_COLUMN_BREAKPOINT = 576;
+var MOBILE_COLUMN_DIALOG_ID = "mobile-column-actions-dialog";
+var MOBILE_COLUMN_DIALOG_TITLE_ID = "mobile-column-actions-title";
+
+function mobileColumnHeaders(manager) {
+  return Array.from(
+    document.querySelectorAll(manager.selectors.tableHeaders),
+  ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1");
+}
+
+function mobileColumnMetaText(th) {
+  var parts = [];
+  if (th.dataset.columnType) {
+    parts.push(th.dataset.columnType);
+  }
+  if (th.dataset.isPk === "1") {
+    parts.push("pk");
+  }
+  if (th.dataset.columnNotNull === "1") {
+    parts.push("not null");
+  }
+  return parts.join(", ");
+}
+
+function createMobileColumnActionNode(itemConfig, closeDialog) {
+  var actionNode;
+  if (itemConfig.href) {
+    actionNode = document.createElement("a");
+    actionNode.href = itemConfig.href;
+  } else {
+    actionNode = document.createElement("button");
+    actionNode.type = "button";
+  }
+  actionNode.textContent = itemConfig.label;
+
+  if (itemConfig.onClick) {
+    actionNode.addEventListener("click", function (ev) {
+      try {
+        itemConfig.onClick.call(actionNode, ev);
+      } finally {
+        closeDialog({ restoreFocus: false });
+      }
+    });
+  }
+
+  return actionNode;
+}
+
+function initMobileColumnActions(manager) {
+  var triggerButton = document.querySelector(".column-actions-mobile");
+  if (!triggerButton) {
+    return;
+  }
+
+  if (
+    !window.URLSearchParams ||
+    !window.HTMLDialogElement ||
+    !manager.columnActions
+  ) {
+    triggerButton.style.display = "none";
+    return;
+  }
+
+  if (!mobileColumnHeaders(manager).length) {
+    triggerButton.style.display = "none";
+    return;
+  }
+
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
+  dialog.className = "mobile-column-actions-dialog";
+  dialog.id = MOBILE_COLUMN_DIALOG_ID;
+  dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID);
+  dialog.innerHTML = `
+    <div class="modal-header">
+      <span class="modal-title" id="${MOBILE_COLUMN_DIALOG_TITLE_ID}">Column actions</span>
+      <span class="modal-meta"></span>
+    </div>
+    <div class="modal-body list-wrap mobile-column-list"></div>
+    <div class="modal-footer">
+      <span class="footer-info">Tap a column to reveal actions.</span>
+      <button type="button" class="modal-btn modal-btn-ghost mobile-column-actions-done">Done</button>
+    </div>
+  `;
+  document.body.appendChild(modal);
+
+  triggerButton.setAttribute("aria-haspopup", "dialog");
+  triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID);
+  triggerButton.setAttribute("aria-expanded", "false");
+
+  var countEl = dialog.querySelector(".modal-meta");
+  var listWrap = dialog.querySelector(".mobile-column-list");
+  var doneButton = dialog.querySelector(".mobile-column-actions-done");
+  var expandedSectionId = null;
+
+  function updateExpandedSection() {
+    Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => {
+      var controlsId = button.getAttribute("aria-controls");
+      var actionList = dialog.querySelector("#" + controlsId);
+      var isExpanded = controlsId === expandedSectionId;
+      button.setAttribute("aria-expanded", isExpanded ? "true" : "false");
+      actionList.hidden = !isExpanded;
+      actionList.classList.toggle("expanded", isExpanded);
+    });
+  }
+
+  function scrollExpandedSectionIntoView(section) {
+    var sectionTop = section.offsetTop;
+    var sectionBottom = sectionTop + section.offsetHeight;
+    var visibleTop = listWrap.scrollTop;
+    var visibleBottom = visibleTop + listWrap.clientHeight;
+    var sectionHeight = section.offsetHeight;
+
+    if (sectionTop < visibleTop) {
+      listWrap.scrollTop = sectionTop;
+      return;
+    }
+
+    if (sectionBottom <= visibleBottom) {
+      return;
+    }
+
+    if (sectionHeight <= listWrap.clientHeight) {
+      listWrap.scrollTop = sectionBottom - listWrap.clientHeight;
+    } else {
+      listWrap.scrollTop = sectionTop;
+    }
+  }
+
+  function closeDialog(options) {
+    modal.close(options);
+  }
+
+  function renderDialog() {
+    var headers = mobileColumnHeaders(manager);
+    if (!headers.length) {
+      closeDialog({ restoreFocus: false });
+      triggerButton.style.display = "none";
+      return false;
+    }
+
+    if (
+      !headers.some(
+        (_th, index) => `mobile-column-actions-${index}` === expandedSectionId,
+      )
+    ) {
+      expandedSectionId = null;
+    }
+
+    countEl.textContent = `${headers.length} column${
+      headers.length === 1 ? "" : "s"
+    }`;
+    listWrap.innerHTML = "";
+
+    if (manager.columnActions.shouldShowShowAllColumns()) {
+      var topActions = document.createElement("div");
+      topActions.className = "mobile-column-top-actions";
+
+      var showAllColumns = document.createElement("a");
+      showAllColumns.className =
+        "modal-btn modal-btn-ghost mobile-column-top-action";
+      showAllColumns.href = manager.columnActions.showAllColumnsUrl();
+      showAllColumns.textContent = "Show all columns";
+
+      topActions.appendChild(showAllColumns);
+      listWrap.appendChild(topActions);
+    }
+
+    headers.forEach((th, index) => {
+      var sectionId = `mobile-column-actions-${index}`;
+      var actionState = manager.columnActions.buildColumnActionState(th, {
+        includeChooseColumns: false,
+        includeShowAllColumns: false,
+      });
+      var section = document.createElement("section");
+      section.className = "mobile-column-section";
+
+      var headerButton = document.createElement("button");
+      headerButton.type = "button";
+      headerButton.className = "col-header";
+      headerButton.setAttribute("aria-controls", sectionId);
+      headerButton.setAttribute("aria-expanded", "false");
+
+      var headerText = document.createElement("span");
+      headerText.className = "mobile-column-header-text";
+
+      var name = document.createElement("span");
+      name.className = "mobile-column-name";
+      name.textContent = th.dataset.column;
+      headerText.appendChild(name);
+
+      var metaText = mobileColumnMetaText(th);
+      if (metaText) {
+        var meta = document.createElement("span");
+        meta.className = "mobile-column-meta";
+        meta.textContent = metaText;
+        headerText.appendChild(meta);
+      }
+
+      var chevron = document.createElement("span");
+      chevron.className = "mobile-column-chevron";
+      chevron.setAttribute("aria-hidden", "true");
+      chevron.textContent = "▾";
+
+      headerButton.appendChild(headerText);
+      headerButton.appendChild(chevron);
+      headerButton.addEventListener("click", function () {
+        expandedSectionId = expandedSectionId === sectionId ? null : sectionId;
+        updateExpandedSection();
+        if (expandedSectionId === sectionId) {
+          scrollExpandedSectionIntoView(section);
+        }
+      });
+
+      var actionContainer = document.createElement("div");
+      actionContainer.id = sectionId;
+      actionContainer.className = "col-actions";
+      actionContainer.hidden = true;
+
+      if (actionState.columnDescription) {
+        var description = document.createElement("p");
+        description.className = "mobile-column-description";
+        description.textContent = actionState.columnDescription;
+        actionContainer.appendChild(description);
+      }
+
+      if (actionState.actionItems.length) {
+        var actionList = document.createElement("ul");
+        actionState.actionItems.forEach((itemConfig) => {
+          var actionItem = document.createElement("li");
+          actionItem.appendChild(
+            createMobileColumnActionNode(itemConfig, closeDialog),
+          );
+          actionList.appendChild(actionItem);
+        });
+        actionContainer.appendChild(actionList);
+      } else {
+        var noActions = document.createElement("p");
+        noActions.className = "mobile-column-no-actions";
+        noActions.textContent = "No actions available";
+        actionContainer.appendChild(noActions);
+      }
+
+      section.appendChild(headerButton);
+      section.appendChild(actionContainer);
+      listWrap.appendChild(section);
+    });
+
+    updateExpandedSection();
+    return true;
+  }
+
+  function openDialog() {
+    if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT) {
+      return;
+    }
+    if (!renderDialog()) {
+      return;
+    }
+    modal.show({ returnFocusTo: triggerButton });
+    triggerButton.setAttribute("aria-expanded", "true");
+    var focusTarget =
+      dialog.querySelector(".mobile-column-top-action") ||
+      dialog.querySelector(".col-header") ||
+      doneButton;
+    focusTarget.focus();
+  }
+
+  triggerButton.addEventListener("click", function () {
+    if (dialog.open) {
+      closeDialog();
+    } else {
+      openDialog();
+    }
+  });
+
+  doneButton.addEventListener("click", function () {
+    closeDialog();
+  });
+
+  dialog.addEventListener("close", function () {
+    triggerButton.setAttribute("aria-expanded", "false");
+  });
+
+  window.addEventListener("resize", function () {
+    if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT && dialog.open) {
+      closeDialog({ restoreFocus: false });
+    }
+  });
+}
+
+document.addEventListener("datasette_init", function (evt) {
+  initMobileColumnActions(evt.detail);
+});

datasette/static/modal.js

@@ -0,0 +1,167 @@
+// Shared lifecycle for native modal dialogs.
+(() => {
+  class DatasetteModal extends HTMLElement {
+    constructor() {
+      super();
+      this.beforeClose = null;
+      this._busy = false;
+      this._restoreFocus = true;
+      this._returnFocusTo = null;
+      this._escapeCleanup = null;
+      this._escapeTimer = null;
+    }
+
+    static create() {
+      const modal = document.createElement("datasette-modal");
+      modal.appendChild(document.createElement("dialog"));
+      return modal;
+    }
+
+    get dialog() {
+      return this.querySelector(":scope > dialog");
+    }
+
+    get busy() {
+      return this._busy;
+    }
+
+    set busy(value) {
+      this._busy = !!value;
+      if (this.dialog) {
+        this.dialog.setAttribute("aria-busy", String(this._busy));
+      }
+    }
+
+    connectedCallback() {
+      const dialog = this.dialog;
+      if (!dialog) return;
+      dialog.classList.add("datasette-modal");
+      this._listeners?.abort();
+      this._listeners = new AbortController();
+      const options = { signal: this._listeners.signal };
+      let backdropPointerDown = false;
+      const outside = (event) => {
+        const rect = dialog.getBoundingClientRect();
+        return (
+          event.target === dialog &&
+          (event.clientX < rect.left ||
+            event.clientX > rect.right ||
+            event.clientY < rect.top ||
+            event.clientY > rect.bottom)
+        );
+      };
+      dialog.addEventListener(
+        "pointerdown",
+        (event) => {
+          backdropPointerDown = outside(event);
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "click",
+        (event) => {
+          if (backdropPointerDown && outside(event))
+            this.requestClose("backdrop");
+          backdropPointerDown = false;
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "keydown",
+        (event) => {
+          if (event.key !== "Escape" || event.defaultPrevented) return;
+          // A nested native dialog or plugin picker gets first refusal.
+          if (event.target.closest("dialog") !== dialog) return;
+          event.preventDefault();
+          if (this.busy || this._escapeCleanup || this._escapeTimer !== null)
+            return;
+          // Safari can otherwise use this Escape press to cancel confirm() too.
+          // Only keyboard dismissals wait for keyup; native cancel events needn't.
+          const onKeyup = (up) => {
+            if (up.key !== "Escape") return;
+            this._escapeCleanup();
+            this._escapeCleanup = null;
+            this._escapeTimer = setTimeout(() => {
+              this._escapeTimer = null;
+              this.requestClose("escape");
+            }, 0);
+          };
+          this.ownerDocument.addEventListener("keyup", onKeyup, true);
+          this._escapeCleanup = () =>
+            this.ownerDocument.removeEventListener("keyup", onKeyup, true);
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "cancel",
+        (event) => {
+          if (event.target !== dialog) return;
+          event.preventDefault();
+          if (!this._escapeCleanup && this._escapeTimer === null)
+            this.requestClose("escape");
+        },
+        options,
+      );
+      dialog.addEventListener(
+        "close",
+        (event) => {
+          if (event.target !== dialog || dialog.open) return;
+          this._clearPendingClose();
+          this.busy = false;
+          if (this._restoreFocus && this._returnFocusTo?.isConnected) {
+            // Menu actions may have become hidden while the dialog was open.
+            const details = this._returnFocusTo.closest("details:not([open])");
+            const target =
+              details?.querySelector("summary") || this._returnFocusTo;
+            target.focus({ preventScroll: true });
+          }
+          this._returnFocusTo = null;
+        },
+        options,
+      );
+    }
+
+    disconnectedCallback() {
+      this._listeners?.abort();
+      this._clearPendingClose();
+      this._returnFocusTo = null;
+      if (this.dialog?.open) this.dialog.close();
+      this.busy = false;
+    }
+
+    _clearPendingClose() {
+      this._escapeCleanup?.();
+      this._escapeCleanup = null;
+      clearTimeout(this._escapeTimer);
+      this._escapeTimer = null;
+    }
+
+    show({ returnFocusTo, initialFocus } = {}) {
+      const dialog = this.dialog;
+      if (!dialog.open) {
+        this._clearPendingClose();
+        this._returnFocusTo = returnFocusTo || this.ownerDocument.activeElement;
+        this._restoreFocus = true;
+        dialog.showModal();
+      }
+      if (typeof initialFocus === "function") initialFocus();
+      else initialFocus?.focus();
+    }
+
+    requestClose(source = "cancel") {
+      if (!this.dialog.open || this.busy) return false;
+      if (this.beforeClose && this.beforeClose(source) === false) return false;
+      this.close();
+      return true;
+    }
+
+    close({ restoreFocus = true } = {}) {
+      this._clearPendingClose();
+      this._restoreFocus = restoreFocus;
+      this.dialog.close();
+    }
+  }
+
+  customElements.define("datasette-modal", DatasetteModal);
+  window.DatasetteModal = DatasetteModal;
+})();

datasette/static/navigation-search.js

@@ -1,195 +1,68 @@
+let navigationSearchInstanceCounter = 0;
+
 class NavigationSearch extends HTMLElement {
   constructor() {
     super();
-    this.attachShadow({ mode: "open" });
+    this.instanceId = ++navigationSearchInstanceCounter;
+    this.inputId = `navigation-search-input-${this.instanceId}`;
+    this.instructionsId = `navigation-search-instructions-${this.instanceId}`;
+    this.listboxId = `navigation-search-results-${this.instanceId}`;
+    this.recentHeadingId = `navigation-search-recent-${this.instanceId}`;
+    this.statusId = `navigation-search-status-${this.instanceId}`;
+    this.titleId = `navigation-search-title-${this.instanceId}`;
     this.selectedIndex = -1;
     this.matches = [];
+    this.renderedMatches = [];
     this.debounceTimer = null;
+  }
 
+  connectedCallback() {
+    if (this._initialized) return;
+    this._initialized = true;
     this.render();
     this.setupEventListeners();
   }
 
   render() {
-    this.shadowRoot.innerHTML = `
+    this.innerHTML = `
-            <style>
+      <datasette-modal><dialog aria-modal="true" aria-labelledby="${this.titleId}">
-                :host {
-                    display: contents;
-                }
-
-                dialog {
-                    border: none;
-                    border-radius: 0.75rem;
-                    padding: 0;
-                    max-width: 90vw;
-                    width: 600px;
-                    max-height: 80vh;
-                    box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04);
-                    animation: slideIn 0.2s ease-out;
-                }
-
-                dialog::backdrop {
-                    background: rgba(0, 0, 0, 0.5);
-                    backdrop-filter: blur(4px);
-                    animation: fadeIn 0.2s ease-out;
-                }
-
-                @keyframes slideIn {
-                    from {
-                        opacity: 0;
-                        transform: translateY(-20px) scale(0.95);
-                    }
-                    to {
-                        opacity: 1;
-                        transform: translateY(0) scale(1);
-                    }
-                }
-
-                @keyframes fadeIn {
-                    from { opacity: 0; }
-                    to { opacity: 1; }
-                }
-
-                .search-container {
-                    display: flex;
-                    flex-direction: column;
-                    height: 100%;
-                }
-
-                .search-input-wrapper {
-                    padding: 1.25rem;
-                    border-bottom: 1px solid #e5e7eb;
-                }
-
-                .search-input {
-                    width: 100%;
-                    padding: 0.75rem 1rem;
-                    font-size: 1rem;
-                    border: 2px solid #e5e7eb;
-                    border-radius: 0.5rem;
-                    outline: none;
-                    transition: border-color 0.2s;
-                    box-sizing: border-box;
-                }
-
-                .search-input:focus {
-                    border-color: #2563eb;
-                }
-
-                .results-container {
-                    overflow-y: auto;
-                    height: calc(80vh - 180px);
-                    padding: 0.5rem;
-                }
-
-                .result-item {
-                    padding: 0.875rem 1rem;
-                    cursor: pointer;
-                    border-radius: 0.5rem;
-                    transition: background-color 0.15s;
-                    display: flex;
-                    align-items: center;
-                    gap: 0.75rem;
-                }
-
-                .result-item:hover {
-                    background-color: #f3f4f6;
-                }
-
-                .result-item.selected {
-                    background-color: #dbeafe;
-                }
-
-                .result-name {
-                    font-weight: 500;
-                    color: #111827;
-                }
-
-                .result-url {
-                    font-size: 0.875rem;
-                    color: #6b7280;
-                }
-
-                .no-results {
-                    padding: 2rem;
-                    text-align: center;
-                    color: #6b7280;
-                }
-
-                .hint-text {
-                    padding: 0.75rem 1.25rem;
-                    font-size: 0.875rem;
-                    color: #6b7280;
-                    border-top: 1px solid #e5e7eb;
-                    display: flex;
-                    gap: 1rem;
-                    flex-wrap: wrap;
-                }
-
-                .hint-text kbd {
-                    background: #f3f4f6;
-                    padding: 0.125rem 0.375rem;
-                    border-radius: 0.25rem;
-                    font-size: 0.75rem;
-                    border: 1px solid #d1d5db;
-                    font-family: monospace;
-                }
-
-                /* Mobile optimizations */
-                @media (max-width: 640px) {
-                    dialog {
-                        width: 95vw;
-                        max-height: 85vh;
-                        border-radius: 0.5rem;
-                    }
-
-                    .search-input-wrapper {
-                        padding: 1rem;
-                    }
-
-                    .search-input {
-                        font-size: 16px; /* Prevents zoom on iOS */
-                    }
-
-                    .result-item {
-                        padding: 1rem 0.75rem;
-                    }
-
-                    .hint-text {
-                        font-size: 0.8rem;
-                        padding: 0.5rem 1rem;
-                    }
-                }
-            </style>
-
-            <dialog>
                 <div class="search-container">
+                    <h2 id="${this.titleId}" class="visually-hidden">Jump to</h2>
+                    <p id="${this.instructionsId}" class="visually-hidden">Type to search. Use up and down arrow keys to move through results, Enter to select a result, and Escape to close this menu.</p>
+                    <div id="${this.statusId}" class="visually-hidden" aria-live="polite" aria-atomic="true"></div>
                     <div class="search-input-wrapper">
                         <input 
+                            id="${this.inputId}"
                             type="text" 
                             class="search-input" 
-                            placeholder="Search..."
+                            placeholder="Jump to..."
-                            aria-label="Search navigation"
+                            aria-label="Jump to"
+                            aria-describedby="${this.instructionsId}"
+                            role="combobox"
+                            aria-autocomplete="list"
+                            aria-controls="${this.listboxId}"
+                            aria-expanded="false"
                             autocomplete="off"
                             spellcheck="false"
                         >
+                        <button type="button" class="close-search" aria-label="Close jump menu">&times;</button>
                     </div>
-                    <div class="results-container" role="listbox"></div>
+                    <div class="modal-body results-container"></div>
                     <div class="hint-text">
                         <span><kbd>↑</kbd> <kbd>↓</kbd> Navigate</span>
                         <span><kbd>Enter</kbd> Select</span>
                         <span><kbd>Esc</kbd> Close</span>
                     </div>
                 </div>
-            </dialog>
+            </dialog></datasette-modal>
         `;
   }
 
   setupEventListeners() {
-    const dialog = this.shadowRoot.querySelector("dialog");
+    const dialog = this.querySelector("dialog");
-    const input = this.shadowRoot.querySelector(".search-input");
+    const input = this.querySelector(".search-input");
-    const resultsContainer =
+    const closeButton = this.querySelector(".close-search");
-      this.shadowRoot.querySelector(".results-container");
+    const resultsContainer = this.querySelector(".results-container");
 
     // Global keyboard listener for "/"
     document.addEventListener("keydown", (e) => {
@@ -199,6 +72,17 @@ class NavigationSearch extends HTMLElement {
       }
     });
 
+    document.addEventListener("click", (e) => {
+      const trigger = e.target.closest("[data-navigation-search-open]");
+      if (trigger) {
+        e.preventDefault();
+        const details = trigger.closest("details");
+        const restoreTarget = details?.querySelector("summary") || trigger;
+        details?.removeAttribute("open");
+        this.openMenu(restoreTarget);
+      }
+    });
+
     // Input event
     input.addEventListener("input", (e) => {
       this.handleSearch(e.target.value);
@@ -215,13 +99,22 @@ class NavigationSearch extends HTMLElement {
       } else if (e.key === "Enter") {
         e.preventDefault();
         this.selectCurrentItem();
-      } else if (e.key === "Escape") {
-        this.closeMenu();
       }
     });
 
+    closeButton.addEventListener("click", () => {
+      this.closeMenu();
+    });
+
     // Click on result item
     resultsContainer.addEventListener("click", (e) => {
+      const clearRecent = e.target.closest("[data-clear-recent-items]");
+      if (clearRecent) {
+        e.preventDefault();
+        this.clearRecentItems();
+        return;
+      }
+
       const item = e.target.closest(".result-item");
       if (item) {
         const index = parseInt(item.dataset.index);
@@ -229,11 +122,8 @@ class NavigationSearch extends HTMLElement {
       }
     });
 
-    // Close on backdrop click
+    dialog.addEventListener("close", () => {
-    dialog.addEventListener("click", (e) => {
+      this.onMenuClosed();
-      if (e.target === dialog) {
-        this.closeMenu();
-      }
     });
 
     // Initial load
@@ -250,6 +140,93 @@ class NavigationSearch extends HTMLElement {
     );
   }
 
+  setElementAttribute(element, name, value) {
+    if (!element) {
+      return;
+    }
+    if (typeof element.setAttribute === "function") {
+      element.setAttribute(name, value);
+    } else {
+      element[name] = String(value);
+    }
+  }
+
+  removeElementAttribute(element, name) {
+    if (!element) {
+      return;
+    }
+    if (typeof element.removeAttribute === "function") {
+      element.removeAttribute(name);
+    } else {
+      delete element[name];
+    }
+  }
+
+  setNavigationTriggersExpanded(expanded) {
+    if (typeof document.querySelectorAll !== "function") {
+      return;
+    }
+    document
+      .querySelectorAll("[data-navigation-search-open]")
+      .forEach((trigger) => {
+        this.setElementAttribute(
+          trigger,
+          "aria-expanded",
+          expanded ? "true" : "false",
+        );
+      });
+  }
+
+  resultOptionId(index) {
+    return `${this.listboxId}-option-${index}`;
+  }
+
+  updateComboboxState() {
+    const dialog = this.querySelector("dialog");
+    const input = this.querySelector(".search-input");
+    const matches = this.renderedMatches || [];
+    this.setElementAttribute(
+      input,
+      "aria-expanded",
+      dialog && dialog.open && matches.length > 0 ? "true" : "false",
+    );
+
+    if (
+      dialog &&
+      dialog.open &&
+      this.selectedIndex >= 0 &&
+      this.selectedIndex < matches.length
+    ) {
+      this.setElementAttribute(
+        input,
+        "aria-activedescendant",
+        this.resultOptionId(this.selectedIndex),
+      );
+    } else {
+      this.removeElementAttribute(input, "aria-activedescendant");
+    }
+  }
+
+  setStatus(message) {
+    const status = this.querySelector(`#${this.statusId}`);
+    if (status) {
+      status.textContent = message || "";
+    }
+  }
+
+  resultsStatus(count, truncated) {
+    if (truncated) {
+      return "More than 100 results. Keep typing to narrow the list.";
+    }
+    if (count === 0) {
+      return "No results found.";
+    }
+    if (count === 1) {
+      return "1 result.";
+    }
+    return `${count} results.`;
+  }
+
   loadInitialData() {
     const itemsAttr = this.getAttribute("items");
     if (itemsAttr) {
@@ -266,6 +243,11 @@ class NavigationSearch extends HTMLElement {
 
   handleSearch(query) {
     clearTimeout(this.debounceTimer);
+    if (query.trim()) {
+      this.setStatus("Searching...");
+    } else {
+      this.setStatus("");
+    }
 
     this.debounceTimer = setTimeout(() => {
       const url = this.getAttribute("url");
@@ -288,65 +270,262 @@ class NavigationSearch extends HTMLElement {
       this.matches = data.matches || [];
       this.selectedIndex = this.matches.length > 0 ? 0 : -1;
       this.renderResults();
+      if (query.trim()) {
+        this.setStatus(this.resultsStatus(this.matches.length, data.truncated));
+      } else {
+        this.setStatus("");
+      }
     } catch (e) {
       console.error("Failed to fetch search results:", e);
       this.matches = [];
       this.renderResults();
+      this.setStatus("Search failed.");
     }
   }
 
   filterLocalItems(query) {
     if (!query.trim()) {
-      this.matches = [];
+      this.matches = this.allItems || [];
     } else {
       const lowerQuery = query.toLowerCase();
       this.matches = (this.allItems || []).filter(
         (item) =>
           item.name.toLowerCase().includes(lowerQuery) ||
+          (item.display_name || "").toLowerCase().includes(lowerQuery) ||
           item.url.toLowerCase().includes(lowerQuery),
       );
     }
     this.selectedIndex = this.matches.length > 0 ? 0 : -1;
     this.renderResults();
+    if (query.trim()) {
+      this.setStatus(this.resultsStatus(this.matches.length, false));
+    } else {
+      this.setStatus("");
+    }
   }
 
-  renderResults() {
+  recentItemsStorageKey() {
-    const container = this.shadowRoot.querySelector(".results-container");
+    return "datasette.navigationSearch.recentItems";
-    const input = this.shadowRoot.querySelector(".search-input");
+  }
 
-    if (this.matches.length === 0) {
+  loadRecentItems() {
-      const message = input.value.trim()
+    if (typeof localStorage === "undefined") {
-        ? "No results found"
+      return [];
-        : "Start typing to search...";
+    }
-      container.innerHTML = `<div class="no-results">${message}</div>`;
+
+    try {
+      const raw = localStorage.getItem(this.recentItemsStorageKey());
+      if (!raw) {
+        return [];
+      }
+      const parsed = JSON.parse(raw);
+      if (!Array.isArray(parsed)) {
+        return [];
+      }
+      return parsed
+        .filter((item) => item && item.name && item.url)
+        .map((item) => ({
+          name: String(item.name),
+          display_name: item.display_name ? String(item.display_name) : "",
+          url: String(item.url),
+          type: item.type ? String(item.type) : "",
+          description: item.description ? String(item.description) : "",
+        }))
+        .slice(0, 5);
+    } catch (e) {
+      return [];
+    }
+  }
+
+  saveRecentItem(match) {
+    if (
+      typeof localStorage === "undefined" ||
+      !match ||
+      !match.name ||
+      !match.url
+    ) {
       return;
     }
 
-    container.innerHTML = this.matches
+    try {
-      .map(
+      const item = {
-        (match, index) => `
+        name: String(match.name),
+        display_name: match.display_name ? String(match.display_name) : "",
+        url: String(match.url),
+        type: match.type ? String(match.type) : "",
+        description: match.description ? String(match.description) : "",
+      };
+      const recentItems = this.loadRecentItems().filter(
+        (recentItem) => recentItem.url !== item.url,
+      );
+      localStorage.setItem(
+        this.recentItemsStorageKey(),
+        JSON.stringify([item, ...recentItems].slice(0, 5)),
+      );
+    } catch (e) {
+      // localStorage may be unavailable, full, or disabled.
+    }
+  }
+
+  clearRecentItems() {
+    if (typeof localStorage === "undefined") {
+      return;
+    }
+
+    try {
+      localStorage.removeItem(this.recentItemsStorageKey());
+    } catch (e) {
+      localStorage.setItem(this.recentItemsStorageKey(), "[]");
+    }
+    this.renderResults();
+    this.setStatus("Recent items cleared.");
+  }
+
+  jumpSections() {
+    const manager = window.__DATASETTE__;
+    if (!manager || typeof manager.makeJumpSections !== "function") {
+      return [];
+    }
+    const sections = manager.makeJumpSections({
+      navigationSearch: this,
+    });
+    return Array.isArray(sections)
+      ? sections.filter(
+          (section) => section && typeof section.render === "function",
+        )
+      : [];
+  }
+
+  jumpSectionsHtml(jumpSections) {
+    return jumpSections
+      .map((section, index) => {
+        const id = section.id
+          ? ` data-jump-section-id="${this.escapeHtml(section.id)}"`
+          : "";
+        return `<div class="jump-start-content" data-jump-section-index="${index}"${id}></div>`;
+      })
+      .join("");
+  }
+
+  renderJumpSections(container, jumpSections) {
+    jumpSections.forEach((section, index) => {
+      const node = container.querySelector(
+        `[data-jump-section-index="${index}"]`,
+      );
+      if (!node) {
+        return;
+      }
+      section.render(node, {
+        navigationSearch: this,
+        container,
+        input: this.querySelector(".search-input"),
+      });
+    });
+  }
+
+  resultItemHtml(match, index) {
+    const displayName = match.display_name || match.name;
+    const label =
+      match.display_name && match.display_name !== match.name
+        ? `<div class="result-label">${this.escapeHtml(match.name)}</div>`
+        : "";
+    const type = match.type
+      ? `<div class="result-type">${this.escapeHtml(match.type)}</div>`
+      : "";
+    const description = match.description
+      ? `<div class="result-description">${this.escapeHtml(
+          match.description,
+        )}</div>`
+      : "";
+    return `
             <div
-                class="result-item ${
+                id="${this.resultOptionId(index)}"
-                  index === this.selectedIndex ? "selected" : ""
+                class="result-item ${index === this.selectedIndex ? "selected" : ""}"
-                }" 
                 data-index="${index}"
                 role="option"
                 aria-selected="${index === this.selectedIndex}"
             >
                 <div>
-                    <div class="result-name">${this.escapeHtml(
+                    ${type}
-                      match.name,
+                    <div class="result-name">${this.escapeHtml(displayName)}</div>
-                    )}</div>
+                    ${label}
                     <div class="result-url">${this.escapeHtml(match.url)}</div>
+                    ${description}
                 </div>
             </div>
-        `,
+        `;
+  }
+
+  renderResults() {
+    const container = this.querySelector(".results-container");
+    const input = this.querySelector(".search-input");
+    const showStartContent = !input.value.trim();
+    const jumpSections = showStartContent ? this.jumpSections() : [];
+    const startBlock = showStartContent
+      ? this.jumpSectionsHtml(jumpSections)
+      : "";
+    const recentItems = showStartContent ? this.loadRecentItems() : [];
+    const defaultMatches = showStartContent ? [] : this.matches;
+    const renderedMatches = [...recentItems, ...defaultMatches];
+    this.renderedMatches = renderedMatches;
+    const emptyListbox = `<div id="${this.listboxId}" class="results-list" role="listbox" aria-label="Jump results"></div>`;
+
+    if (renderedMatches.length) {
+      if (
+        this.selectedIndex < 0 ||
+        this.selectedIndex >= renderedMatches.length
+      ) {
+        this.selectedIndex = 0;
+      }
+    } else {
+      this.selectedIndex = -1;
+    }
+
+    if (renderedMatches.length === 0) {
+      if (startBlock) {
+        container.innerHTML = startBlock + emptyListbox;
+        this.renderJumpSections(container, jumpSections);
+      } else if (showStartContent) {
+        container.innerHTML = emptyListbox;
+      } else {
+        const message = input.value.trim()
+          ? "No results found"
+          : "Start typing to search...";
+        container.innerHTML = `${emptyListbox}<div class="no-results">${message}</div>`;
+      }
+      this.updateComboboxState();
+      return;
+    }
+
+    const recentHeading = recentItems.length
+      ? `<div class="results-heading" id="${this.recentHeadingId}">Recent</div>`
+      : "";
+    const recentGroup = recentItems.length
+      ? `<div role="group" aria-labelledby="${this.recentHeadingId}">${recentItems
+          .map((match, index) => this.resultItemHtml(match, index))
+          .join("")}</div>`
+      : "";
+    const recentActions = recentItems.length
+      ? `<div class="recent-actions"><button type="button" class="clear-recent" data-clear-recent-items>Clear recent</button></div>`
+      : "";
+    const defaultHtml = defaultMatches
+      .map((match, index) =>
+        this.resultItemHtml(match, recentItems.length + index),
       )
       .join("");
+    container.innerHTML =
+      startBlock +
+      recentHeading +
+      `<div id="${this.listboxId}" class="results-list" role="listbox" aria-label="Jump results">${recentGroup}${defaultHtml}</div>` +
+      recentActions;
+    this.renderJumpSections(container, jumpSections);
+    this.updateComboboxState();
 
     // Scroll selected item into view
     if (this.selectedIndex >= 0) {
-      const selectedItem = container.children[this.selectedIndex];
+      const selectedItem = container.querySelector(
+        `.result-item[data-index="${this.selectedIndex}"]`,
+      );
       if (selectedItem) {
         selectedItem.scrollIntoView({ block: "nearest" });
       }
@@ -354,22 +533,27 @@ class NavigationSearch extends HTMLElement {
   }
 
   moveSelection(direction) {
+    const matches = this.renderedMatches || this.matches;
     const newIndex = this.selectedIndex + direction;
-    if (newIndex >= 0 && newIndex < this.matches.length) {
+    if (newIndex >= 0 && newIndex < matches.length) {
       this.selectedIndex = newIndex;
       this.renderResults();
     }
   }
 
   selectCurrentItem() {
-    if (this.selectedIndex >= 0 && this.selectedIndex < this.matches.length) {
+    const matches = this.renderedMatches || this.matches;
+    if (this.selectedIndex >= 0 && this.selectedIndex < matches.length) {
       this.selectItem(this.selectedIndex);
     }
   }
 
   selectItem(index) {
-    const match = this.matches[index];
+    const matches = this.renderedMatches || this.matches;
+    const match = matches[index];
     if (match) {
+      this.saveRecentItem(match);
+
       // Dispatch custom event
       this.dispatchEvent(
         new CustomEvent("select", {
@@ -382,32 +566,42 @@ class NavigationSearch extends HTMLElement {
       // Navigate to URL
       window.location.href = match.url;
 
-      this.closeMenu();
+      this.closeMenu({ restoreFocus: false });
     }
   }
 
-  openMenu() {
+  openMenu(returnFocusTo) {
-    const dialog = this.shadowRoot.querySelector("dialog");
+    const input = this.querySelector(".search-input");
-    const input = this.shadowRoot.querySelector(".search-input");
 
-    dialog.showModal();
+    this.querySelector("datasette-modal").show({
+      returnFocusTo,
+      initialFocus: input,
+    });
+    this.setNavigationTriggersExpanded(true);
     input.value = "";
-    input.focus();
 
-    // Reset state - start with no items shown
+    // Reset state, then populate the default jump list.
     this.matches = [];
     this.selectedIndex = -1;
     this.renderResults();
+    this.setStatus("");
   }
 
-  closeMenu() {
+  closeMenu(options = {}) {
-    const dialog = this.shadowRoot.querySelector("dialog");
+    this.querySelector("datasette-modal").close(options);
-    dialog.close();
+  }
+
+  onMenuClosed() {
+    const input = this.querySelector(".search-input");
+    this.setElementAttribute(input, "aria-expanded", "false");
+    this.removeElementAttribute(input, "aria-activedescendant");
+    this.setNavigationTriggersExpanded(false);
+    this.setStatus("");
   }
 
   escapeHtml(text) {
     const div = document.createElement("div");
-    div.textContent = text;
+    div.textContent = text == null ? "" : text;
     return div.innerHTML;
   }
 }

datasette/static/table.js

@@ -1,13 +1,6 @@
 var DROPDOWN_HTML = `<div class="dropdown-menu">
 <div class="hook"></div>
-<ul>
+<ul class="dropdown-actions"></ul>
-  <li><a class="dropdown-sort-asc" href="#">Sort ascending</a></li>
-  <li><a class="dropdown-sort-desc" href="#">Sort descending</a></li>
-  <li><a class="dropdown-facet" href="#">Facet by this</a></li>
-  <li><a class="dropdown-hide-column" href="#">Hide this column</a></li>
-  <li><a class="dropdown-show-all-columns" href="#">Show all columns</a></li>
-  <li><a class="dropdown-not-blank" href="#">Show not-blank rows</a></li>
-</ul>
 <p class="dropdown-column-type"></p>
 <p class="dropdown-column-description"></p>
 </div>`;
@@ -17,19 +10,17 @@ var DROPDOWN_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="14" heig
   <path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1-2-2 2 2 0 0 1 2-2h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 2-2 2 2 0 0 1 2 2v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-.09a1.65 1.65 0 0 0-1.51 1z"></path>
 </svg>`;
 
-/** Main initialization function for Datasette Table interactions */
+var SET_COLUMN_TYPE_DIALOG_ID = "set-column-type-dialog";
-const initDatasetteTable = function (manager) {
+var setColumnTypeDialogState = null;
-  // Feature detection
-  if (!window.URLSearchParams) {
-    return;
-  }
 function getParams() {
   return new URLSearchParams(location.search);
 }
+
 function paramsToUrl(params) {
   var s = params.toString();
   return s ? "?" + s : location.pathname;
 }
+
 function sortDescUrl(column) {
   var params = getParams();
   params.set("_sort_desc", column);
@@ -37,6 +28,7 @@ const initDatasetteTable = function (manager) {
   params.delete("_next");
   return paramsToUrl(params);
 }
+
 function sortAscUrl(column) {
   var params = getParams();
   params.set("_sort", column);
@@ -44,27 +36,473 @@ const initDatasetteTable = function (manager) {
   params.delete("_next");
   return paramsToUrl(params);
 }
+
 function facetUrl(column) {
   var params = getParams();
   params.append("_facet", column);
   return paramsToUrl(params);
 }
+
 function hideColumnUrl(column) {
   var params = getParams();
   params.append("_nocol", column);
   return paramsToUrl(params);
 }
+
 function showAllColumnsUrl() {
   var params = getParams();
   params.delete("_nocol");
   params.delete("_col");
   return paramsToUrl(params);
 }
+
 function notBlankUrl(column) {
   var params = getParams();
   params.set(`${column}__notblank`, "1");
   return paramsToUrl(params);
 }
+
+function getDisplayedFacets() {
+  return Array.from(document.querySelectorAll(".facet-info")).map(
+    (el) => el.dataset.column,
+  );
+}
+
+function getColumnClassName(th) {
+  return Array.from(th.classList).find((className) =>
+    className.startsWith("col-"),
+  );
+}
+
+function getColumnCells(th) {
+  var table = th.closest("table");
+  var columnClassName = getColumnClassName(th);
+  if (!table || !columnClassName) {
+    return [];
+  }
+  return Array.from(table.querySelectorAll("td." + columnClassName));
+}
+
+function getColumnMeta(th) {
+  return {
+    columnName: th.dataset.column,
+    columnNotNull: th.dataset.columnNotNull === "1",
+    columnType: th.dataset.columnType,
+    isPk: th.dataset.isPk === "1",
+  };
+}
+
+function getColumnTypeText(th) {
+  var columnType = th.dataset.columnType;
+  if (!columnType) {
+    return null;
+  }
+  var notNull = th.dataset.columnNotNull === "1" ? " NOT NULL" : "";
+  return `Type: ${columnType.toUpperCase()}${notNull}`;
+}
+
+function getSetColumnTypeData() {
+  return window._setColumnTypeData || null;
+}
+
+function getSetColumnTypeConfig(column) {
+  var data = getSetColumnTypeData();
+  if (!data || !data.columns) {
+    return null;
+  }
+  return data.columns[column] || null;
+}
+
+function canSetColumnType() {
+  return !!(getSetColumnTypeData() && window.HTMLDialogElement && window.fetch);
+}
+
+function setColumnTypeActionLabel(column) {
+  var columnConfig = getSetColumnTypeConfig(column);
+  if (!columnConfig) {
+    return null;
+  }
+  return columnConfig.current
+    ? `Custom type: ${columnConfig.current.type}`
+    : "Set custom type";
+}
+
+function createSetColumnTypeOption(value, name, description, checked) {
+  var label = document.createElement("label");
+  label.className = "set-column-type-option";
+
+  var input = document.createElement("input");
+  input.type = "radio";
+  input.name = "set-column-type-choice";
+  input.value = value;
+  input.checked = checked;
+
+  var content = document.createElement("span");
+  content.className = "set-column-type-option-content";
+
+  var title = document.createElement("span");
+  title.className = "set-column-type-option-name";
+  title.textContent = name;
+
+  var detail = document.createElement("span");
+  detail.className = "set-column-type-option-description";
+  detail.textContent = description;
+
+  content.appendChild(title);
+  content.appendChild(detail);
+  label.appendChild(input);
+  label.appendChild(content);
+  return label;
+}
+
+function setSetColumnTypeDialogBusy(state, isBusy) {
+  state.isBusy = isBusy;
+  state.modal.busy = isBusy;
+  state.saveButton.disabled = isBusy;
+  state.cancelButton.disabled = isBusy;
+  Array.from(
+    state.optionsWrap.querySelectorAll('input[name="set-column-type-choice"]'),
+  ).forEach(function (input) {
+    input.disabled = isBusy;
+  });
+  state.saveButton.textContent = isBusy ? "Saving..." : "Save";
+}
+
+function clearSetColumnTypeDialogError(state) {
+  state.error.hidden = true;
+  state.error.textContent = "";
+}
+
+function showSetColumnTypeDialogError(state, message) {
+  state.error.hidden = false;
+  state.error.textContent = message;
+}
+
+function ensureSetColumnTypeDialog() {
+  if (setColumnTypeDialogState) {
+    return setColumnTypeDialogState;
+  }
+  if (!window.HTMLDialogElement) {
+    return null;
+  }
+
+  var modal = DatasetteModal.create();
+  var dialog = modal.dialog;
+  dialog.id = SET_COLUMN_TYPE_DIALOG_ID;
+  dialog.className = "set-column-type-dialog";
+  dialog.setAttribute("aria-labelledby", "set-column-type-title");
+  dialog.innerHTML = `
+    <div class="modal-header">
+      <span class="modal-title" id="set-column-type-title">Set custom type</span>
+      <span class="modal-meta"></span>
+    </div>
+    <p class="set-column-type-status"></p>
+    <p class="set-column-type-error" hidden></p>
+    <div class="modal-body set-column-type-options"></div>
+    <div class="modal-footer">
+      <span class="footer-info"></span>
+      <button type="button" class="modal-btn modal-btn-ghost set-column-type-cancel">Cancel</button>
+      <button type="button" class="modal-btn modal-btn-primary set-column-type-save">Save</button>
+    </div>
+  `;
+  document.body.appendChild(modal);
+
+  setColumnTypeDialogState = {
+    modal: modal,
+    dialog: dialog,
+    meta: dialog.querySelector(".modal-meta"),
+    status: dialog.querySelector(".set-column-type-status"),
+    error: dialog.querySelector(".set-column-type-error"),
+    optionsWrap: dialog.querySelector(".set-column-type-options"),
+    footerInfo: dialog.querySelector(".footer-info"),
+    cancelButton: dialog.querySelector(".set-column-type-cancel"),
+    saveButton: dialog.querySelector(".set-column-type-save"),
+    currentColumn: null,
+    currentConfig: null,
+    isBusy: false,
+  };
+
+  setColumnTypeDialogState.cancelButton.addEventListener("click", function () {
+    modal.requestClose("cancel");
+  });
+
+  dialog.addEventListener("close", function () {
+    clearSetColumnTypeDialogError(setColumnTypeDialogState);
+    setSetColumnTypeDialogBusy(setColumnTypeDialogState, false);
+  });
+
+  setColumnTypeDialogState.saveButton.addEventListener(
+    "click",
+    async function () {
+      var state = setColumnTypeDialogState;
+      var selected = state.dialog.querySelector(
+        'input[name="set-column-type-choice"]:checked',
+      );
+      var selectedType = selected ? selected.value : "";
+      var currentType = state.currentConfig.current
+        ? state.currentConfig.current.type
+        : "";
+
+      if (selectedType === currentType) {
+        state.modal.close();
+        return;
+      }
+
+      clearSetColumnTypeDialogError(state);
+      setSetColumnTypeDialogBusy(state, true);
+
+      var payload = {
+        column: state.currentColumn,
+        column_type: selectedType ? { type: selectedType } : null,
+      };
+
+      try {
+        var response = await fetch(getSetColumnTypeData().path, {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            Accept: "application/json",
+          },
+          body: JSON.stringify(payload),
+        });
+        var data = await response.json();
+        if (!response.ok || data.ok === false) {
+          var message = (data.errors || ["Request failed"]).join(" ");
+          throw new Error(message);
+        }
+        location.reload();
+      } catch (error) {
+        setSetColumnTypeDialogBusy(state, false);
+        showSetColumnTypeDialogError(state, error.message || "Request failed");
+      }
+    },
+  );
+
+  return setColumnTypeDialogState;
+}
+
+function openSetColumnTypeDialog(th) {
+  var column = th.dataset.column;
+  var columnConfig = getSetColumnTypeConfig(column);
+  if (!columnConfig) {
+    return;
+  }
+
+  var state = ensureSetColumnTypeDialog();
+  if (!state) {
+    return;
+  }
+
+  clearSetColumnTypeDialogError(state);
+  setSetColumnTypeDialogBusy(state, false);
+  state.currentColumn = column;
+  state.currentConfig = columnConfig;
+  state.status.textContent = `Column: ${column}`;
+  state.meta.textContent = getColumnTypeText(th) || "Type unavailable";
+  state.footerInfo.textContent = columnConfig.current
+    ? `Current custom type: ${columnConfig.current.type}`
+    : "No custom type set.";
+  state.optionsWrap.innerHTML = "";
+
+  var currentType = columnConfig.current ? columnConfig.current.type : "";
+  state.optionsWrap.appendChild(
+    createSetColumnTypeOption(
+      "",
+      "No custom type",
+      "Use standard Datasette rendering without a custom type.",
+      currentType === "",
+    ),
+  );
+
+  columnConfig.options.forEach(function (option) {
+    state.optionsWrap.appendChild(
+      createSetColumnTypeOption(
+        option.name,
+        option.name,
+        option.description,
+        option.name === currentType,
+      ),
+    );
+  });
+
+  if (!columnConfig.options.length) {
+    var emptyState = document.createElement("p");
+    emptyState.className = "set-column-type-empty";
+    emptyState.textContent =
+      "No registered custom types are compatible with this SQLite type.";
+    state.optionsWrap.appendChild(emptyState);
+  }
+
+  state.modal.show();
+  var selectedOption = state.dialog.querySelector(
+    'input[name="set-column-type-choice"]:checked',
+  );
+  if (selectedOption) {
+    selectedOption.focus();
+  } else {
+    state.saveButton.focus();
+  }
+}
+
+function canChooseColumns() {
+  return !!(
+    document.querySelector("column-chooser") && window._columnChooserData
+  );
+}
+
+function shouldShowShowAllColumns() {
+  var params = getParams();
+  return params.getAll("_nocol").length || params.getAll("_col").length;
+}
+
+function hasMultipleVisibleColumns(manager) {
+  return (
+    Array.from(
+      document.querySelectorAll(manager.selectors.tableHeaders),
+    ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1")
+      .length > 1
+  );
+}
+
+function buildColumnActionItems(manager, th, options) {
+  options = options || {};
+  var params = getParams();
+  var column = th.dataset.column;
+  var columnActions = [];
+  var isSortable = !!th.querySelector("a");
+  var isFirstColumn = th.parentElement.querySelector("th:first-of-type") === th;
+  var isSinglePk =
+    th.dataset.isPk === "1" &&
+    document.querySelectorAll('th[data-is-pk="1"]').length === 1;
+  var hasBlankValues = getColumnCells(th).some(
+    (el) => el.innerText.trim() === "",
+  );
+
+  if (isSortable && params.get("_sort") !== column) {
+    columnActions.push({
+      label: "Sort ascending",
+      href: sortAscUrl(column),
+    });
+  }
+
+  if (isSortable && params.get("_sort_desc") !== column) {
+    columnActions.push({
+      label: "Sort descending",
+      href: sortDescUrl(column),
+    });
+  }
+
+  if (
+    DATASETTE_ALLOW_FACET &&
+    !isFirstColumn &&
+    !getDisplayedFacets().includes(column) &&
+    !isSinglePk
+  ) {
+    columnActions.push({
+      label: "Facet by this",
+      href: facetUrl(column),
+    });
+  }
+
+  if (options.includeChooseColumns && canChooseColumns()) {
+    columnActions.push({
+      label: "Choose columns",
+      href: "#",
+      onClick:
+        options.onChooseColumns ||
+        function (ev) {
+          ev.preventDefault();
+          openColumnChooser();
+        },
+    });
+  }
+
+  if (canSetColumnType() && getSetColumnTypeConfig(column)) {
+    columnActions.push({
+      label: setColumnTypeActionLabel(column),
+      href: "#",
+      onClick:
+        options.onSetColumnType ||
+        function (ev) {
+          ev.preventDefault();
+          window.setTimeout(function () {
+            openSetColumnTypeDialog(th);
+          }, 0);
+        },
+    });
+  }
+
+  if (th.dataset.isPk !== "1" && hasMultipleVisibleColumns(manager)) {
+    columnActions.push({
+      label: "Hide this column",
+      href: hideColumnUrl(column),
+    });
+  }
+
+  if (options.includeShowAllColumns && shouldShowShowAllColumns()) {
+    columnActions.push({
+      label: "Show all columns",
+      href: showAllColumnsUrl(),
+    });
+  }
+
+  if (params.get(`${column}__notblank`) !== "1" && hasBlankValues) {
+    columnActions.push({
+      label: "Show not-blank rows",
+      href: notBlankUrl(column),
+    });
+  }
+
+  return columnActions.concat(manager.makeColumnActions(getColumnMeta(th)));
+}
+
+function buildColumnActionState(manager, th, options) {
+  return {
+    column: th.dataset.column,
+    columnDescription: th.dataset.columnDescription || null,
+    columnMeta: getColumnMeta(th),
+    columnTypeText: getColumnTypeText(th),
+    actionItems: buildColumnActionItems(manager, th, options),
+  };
+}
+
+function initializeColumnActions(manager) {
+  manager.columnActions = {
+    buildColumnActionState: function (th, options) {
+      return buildColumnActionState(manager, th, options);
+    },
+    buildColumnActionItems: function (th, options) {
+      return buildColumnActionItems(manager, th, options);
+    },
+    canChooseColumns: canChooseColumns,
+    facetUrl: facetUrl,
+    getColumnMeta: getColumnMeta,
+    getColumnTypeText: getColumnTypeText,
+    hideColumnUrl: hideColumnUrl,
+    notBlankUrl: notBlankUrl,
+    shouldShowShowAllColumns: shouldShowShowAllColumns,
+    showAllColumnsUrl: showAllColumnsUrl,
+    sortAscUrl: sortAscUrl,
+    sortDescUrl: sortDescUrl,
+  };
+}
+
+function renderActionLink(itemConfig) {
+  var newLink = document.createElement("a");
+  newLink.textContent = itemConfig.label;
+  newLink.href = itemConfig.href || "#";
+  if (itemConfig.onClick) {
+    newLink.addEventListener("click", itemConfig.onClick);
+  }
+  return newLink;
+}
+
+/** Main initialization function for Datasette Table interactions */
+const initDatasetteTable = function (manager) {
+  // Feature detection
+  if (!window.URLSearchParams) {
+    return;
+  }
   function closeMenu() {
     menu.style.display = "none";
     menu.classList.remove("anim-scale-in");
@@ -96,87 +534,41 @@ const initDatasetteTable = function (manager) {
     var rect = th.getBoundingClientRect();
     var menuTop = rect.bottom + window.scrollY;
     var menuLeft = rect.left + window.scrollX;
-    var column = th.getAttribute("data-column");
+    var actionState = manager.columnActions.buildColumnActionState(th, {
-    var params = getParams();
+      includeChooseColumns: true,
-    var sort = menu.querySelector("a.dropdown-sort-asc");
+      includeShowAllColumns: true,
-    var sortDesc = menu.querySelector("a.dropdown-sort-desc");
+      onChooseColumns: function (ev) {
-    var facetItem = menu.querySelector("a.dropdown-facet");
+        ev.preventDefault();
-    var notBlank = menu.querySelector("a.dropdown-not-blank");
+        closeMenu();
-    var hideColumn = menu.querySelector("a.dropdown-hide-column");
+        openColumnChooser();
-    var showAllColumns = menu.querySelector("a.dropdown-show-all-columns");
+      },
-    if (params.get("_sort") == column) {
+      onSetColumnType: function (ev) {
-      sort.parentNode.style.display = "none";
+        ev.preventDefault();
-    } else {
+        closeMenu();
-      sort.parentNode.style.display = "block";
+        window.setTimeout(function () {
-      sort.setAttribute("href", sortAscUrl(column));
+          openSetColumnTypeDialog(th);
-    }
+        }, 0);
-    if (params.get("_sort_desc") == column) {
+      },
-      sortDesc.parentNode.style.display = "none";
+    });
-    } else {
+    var menuList = menu.querySelector("ul.dropdown-actions");
-      sortDesc.parentNode.style.display = "block";
+    menuList.innerHTML = "";
-      sortDesc.setAttribute("href", sortDescUrl(column));
+    actionState.actionItems.forEach((itemConfig) => {
-    }
+      var menuItem = document.createElement("li");
-    /* Show hide columns options */
+      menuItem.appendChild(renderActionLink(itemConfig));
-    if (params.get("_nocol") || params.get("_col")) {
+      menuList.appendChild(menuItem);
-      showAllColumns.parentNode.style.display = "block";
+    });
-      showAllColumns.setAttribute("href", showAllColumnsUrl());
-    } else {
-      showAllColumns.parentNode.style.display = "none";
-    }
-    if (th.getAttribute("data-is-pk") != "1") {
-      hideColumn.parentNode.style.display = "block";
-      hideColumn.setAttribute("href", hideColumnUrl(column));
-    } else {
-      hideColumn.parentNode.style.display = "none";
-    }
-    /* Only show "Facet by this" if it's not the first column, not selected,
-       not a single PK and the Datasette allow_facet setting is True */
-    var displayedFacets = Array.from(
-      document.querySelectorAll(".facet-info"),
-    ).map((el) => el.dataset.column);
-    var isFirstColumn =
-      th.parentElement.querySelector("th:first-of-type") == th;
-    var isSinglePk =
-      th.getAttribute("data-is-pk") == "1" &&
-      document.querySelectorAll('th[data-is-pk="1"]').length == 1;
-    if (
-      !DATASETTE_ALLOW_FACET ||
-      isFirstColumn ||
-      displayedFacets.includes(column) ||
-      isSinglePk
-    ) {
-      facetItem.parentNode.style.display = "none";
-    } else {
-      facetItem.parentNode.style.display = "block";
-      facetItem.setAttribute("href", facetUrl(column));
-    }
-    /* Show notBlank option if not selected AND at least one visible blank value */
-    var tdsForThisColumn = Array.from(
-      th.closest("table").querySelectorAll("td." + th.className),
-    );
-    if (
-      params.get(`${column}__notblank`) != "1" &&
-      tdsForThisColumn.filter((el) => el.innerText.trim() == "").length
-    ) {
-      notBlank.parentNode.style.display = "block";
-      notBlank.setAttribute("href", notBlankUrl(column));
-    } else {
-      notBlank.parentNode.style.display = "none";
-    }
-    var columnTypeP = menu.querySelector(".dropdown-column-type");
-    var columnType = th.dataset.columnType;
-    var notNull = th.dataset.columnNotNull == 1 ? " NOT NULL" : "";
 
-    if (columnType) {
+    var columnTypeP = menu.querySelector(".dropdown-column-type");
+    if (actionState.columnTypeText) {
       columnTypeP.style.display = "block";
-      columnTypeP.innerText = `Type: ${columnType.toUpperCase()}${notNull}`;
+      columnTypeP.innerText = actionState.columnTypeText;
     } else {
       columnTypeP.style.display = "none";
     }
 
     var columnDescriptionP = menu.querySelector(".dropdown-column-description");
-    if (th.dataset.columnDescription) {
+    if (actionState.columnDescription) {
-      columnDescriptionP.innerText = th.dataset.columnDescription;
+      columnDescriptionP.innerText = actionState.columnDescription;
       columnDescriptionP.style.display = "block";
     } else {
       columnDescriptionP.style.display = "none";
@@ -187,39 +579,6 @@ const initDatasetteTable = function (manager) {
     menu.style.display = "block";
     menu.classList.add("anim-scale-in");
 
-    // Custom menu items on each render
-    // Plugin hook: allow adding JS-based additional menu items
-    const columnActionsPayload = {
-      columnName: th.dataset.column,
-      columnNotNull: th.dataset.columnNotNull === "1",
-      columnType: th.dataset.columnType,
-      isPk: th.dataset.isPk === "1",
-    };
-    const columnItemConfigs = manager.makeColumnActions(columnActionsPayload);
-
-    const menuList = menu.querySelector("ul");
-    columnItemConfigs.forEach((itemConfig) => {
-      // Remove items from previous render. We assume entries have unique labels.
-      const existingItems = menuList.querySelectorAll(`li`);
-      Array.from(existingItems)
-        .filter((item) => item.innerText === itemConfig.label)
-        .forEach((node) => {
-          node.remove();
-        });
-
-      const newLink = document.createElement("a");
-      newLink.textContent = itemConfig.label;
-      newLink.href = itemConfig.href ?? "#";
-      if (itemConfig.onClick) {
-        newLink.onclick = itemConfig.onClick;
-      }
-
-      // Attach new elements to DOM
-      const menuItem = document.createElement("li");
-      menuItem.appendChild(newLink);
-      menuList.appendChild(menuItem);
-    });
-
     // Measure width of menu and adjust position if too far right
     const menuWidth = menu.offsetWidth;
     const windowWidth = window.innerWidth;
@@ -265,32 +624,157 @@ const initDatasetteTable = function (manager) {
   });
 };
 
-/* Add x buttons to the filter rows */
+function filterRowSelector(manager) {
-function addButtonsToFilterRows(manager) {
+  return manager.selectors.filterRows || manager.selectors.filterRow;
-  var x = "✖";
+}
-  var rows = Array.from(
+
-    document.querySelectorAll(manager.selectors.filterRow),
+function filterRowsWithControls(manager) {
+  return Array.from(
+    document.querySelectorAll(filterRowSelector(manager)),
   ).filter((el) => el.querySelector(".filter-op"));
-  rows.forEach((row) => {
+}
-    var a = document.createElement("a");
+
-    a.setAttribute("href", "#");
+function filterRowNumberFromName(name) {
-    a.setAttribute("aria-label", "Remove this filter");
+  var match = name && name.match(/^_filter_column_(\d+)$/);
-    a.style.textDecoration = "none";
+  return match ? parseInt(match[1], 10) : 0;
-    a.innerText = x;
+}
-    a.addEventListener("click", (ev) => {
+
-      ev.preventDefault();
+function nextFilterRowNumber(manager) {
-      let row = ev.target.closest("div");
+  return (
+    filterRowsWithControls(manager).reduce((max, row) => {
+      var column = row.querySelector("select");
+      return Math.max(max, filterRowNumberFromName(column && column.name));
+    }, 0) + 1
+  );
+}
+
+function setFilterRowNumber(row, number) {
+  row.querySelector("select").name = `_filter_column_${number}`;
+  row.querySelector(".filter-op select").name = `_filter_op_${number}`;
+  row.querySelector("input.filter-value").name = `_filter_value_${number}`;
+}
+
+function resetFilterRow(row) {
   row.querySelector("select").value = "";
   row.querySelector(".filter-op select").value = "exact";
   row.querySelector("input.filter-value").value = "";
-      ev.target.closest("a").style.display = "none";
+}
-    });
+
-    row.appendChild(a);
+function updateFilterRowButtons(manager) {
+  var rows = filterRowsWithControls(manager);
+  rows.forEach((row, index) => {
+    var removeButton = row.querySelector(".filter-row-remove");
+    var addButton = row.querySelector(".filter-row-add");
     var column = row.querySelector("select");
-    if (!column.value) {
+    if (removeButton) {
-      a.style.display = "none";
+      removeButton.hidden = index === 0;
+    }
+    if (addButton) {
+      addButton.hidden = index !== rows.length - 1 || !column.value;
+    }
+    var visibleButtonCount = [removeButton, addButton].filter(
+      function (button) {
+        return button && !button.hidden;
+      },
+    ).length;
+    row.classList.toggle(
+      "filter-controls-row-has-buttons",
+      visibleButtonCount > 0,
+    );
+    row.classList.toggle(
+      "filter-controls-row-one-button",
+      visibleButtonCount === 1,
+    );
+    row.classList.toggle(
+      "filter-controls-row-two-buttons",
+      visibleButtonCount === 2,
+    );
+  });
+}
+
+function cloneFilterRow(row) {
+  var clone = row.cloneNode(true);
+  clone.querySelector("select").name = "_filter_column";
+  clone.querySelector(".filter-op select").name = "_filter_op";
+  clone.querySelector("input.filter-value").name = "_filter_value";
+  resetFilterRow(clone);
+  clone
+    .querySelectorAll(".filter-row-icon")
+    .forEach((button) => button.remove());
+  return clone;
+}
+
+var FILTER_REMOVE_ICON_SVG = `<svg class="filter-row-remove-icon" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.1" stroke-linecap="round" stroke-linejoin="round">
+  <path d="M3 6h18"></path>
+  <path d="M8 6V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"></path>
+  <path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"></path>
+  <path d="M10 11v6"></path>
+  <path d="M14 11v6"></path>
+</svg>`;
+
+var FILTER_ADD_ICON_SVG = `<svg class="filter-row-add-icon" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round">
+  <path d="M5 12h14"></path>
+  <path d="M12 5v14"></path>
+</svg>`;
+
+function addFilterRowButtons(row, manager) {
+  var removeButton = document.createElement("button");
+  removeButton.type = "button";
+  removeButton.className = "filter-row-icon filter-row-remove";
+  removeButton.setAttribute("aria-label", "Remove this filter");
+  removeButton.title = "Remove this filter";
+  removeButton.tabIndex = 0;
+  removeButton.innerHTML = FILTER_REMOVE_ICON_SVG;
+  removeButton.addEventListener("click", (ev) => {
+    var row = ev.currentTarget.closest(filterRowSelector(manager));
+    var rows = filterRowsWithControls(manager);
+    var rowIndex = rows.indexOf(row);
+    var focusRow = rows[rowIndex + 1] || rows[rowIndex - 1] || null;
+    row.remove();
+    updateFilterRowButtons(manager);
+    if (focusRow) {
+      var focusTarget =
+        focusRow.querySelector(".filter-row-add:not([hidden])") ||
+        focusRow.querySelector("select");
+      if (focusTarget) {
+        focusTarget.focus();
+      }
     }
   });
+  row.appendChild(removeButton);
+
+  var addButton = document.createElement("button");
+  addButton.type = "button";
+  addButton.className = "filter-row-icon filter-row-add";
+  addButton.setAttribute("aria-label", "Add another filter");
+  addButton.title = "Add another filter";
+  addButton.tabIndex = 0;
+  addButton.innerHTML = FILTER_ADD_ICON_SVG;
+  addButton.addEventListener("click", (ev) => {
+    var row = ev.currentTarget.closest(filterRowSelector(manager));
+    if (row.querySelector("select").name === "_filter_column") {
+      setFilterRowNumber(row, nextFilterRowNumber(manager));
+    }
+    var clone = cloneFilterRow(row);
+    addFilterRowButtons(clone, manager);
+    row.parentNode.insertBefore(clone, row.nextSibling);
+    updateFilterRowButtons(manager);
+    clone.querySelector("select").focus();
+  });
+  row.appendChild(addButton);
+
+  row.querySelector("select").addEventListener("change", () => {
+    updateFilterRowButtons(manager);
+  });
+}
+
+/* Add buttons to the filter rows */
+function addButtonsToFilterRows(manager) {
+  var rows = filterRowsWithControls(manager);
+  rows.forEach((row) => {
+    addFilterRowButtons(row, manager);
+  });
+  updateFilterRowButtons(manager);
 }
 
 /* Set up datalist autocomplete for filter values */
@@ -319,21 +803,101 @@ function initAutocompleteForFilterValues(manager) {
     });
   }
   createDataLists();
-  // When any select with name=_filter_column changes, update the datalist
+  // When any filter column select changes, update the datalist
   document.body.addEventListener("change", function (event) {
-    if (event.target.name === "_filter_column") {
+    if (event.target.name && event.target.name.startsWith("_filter_column")) {
       event.target
-        .closest(manager.selectors.filterRow)
+        .closest(filterRowSelector(manager))
         .querySelector(".filter-value")
         .setAttribute("list", "datalist-" + event.target.value);
     }
   });
 }
 
+/** Open the column-chooser web component */
+function openColumnChooser() {
+  var chooser = document.querySelector("column-chooser");
+  var data = window._columnChooserData;
+  if (!chooser || !data) return;
+
+  var nonPkColumns = data.allColumns.filter(function (col) {
+    return data.primaryKeys.indexOf(col) === -1;
+  });
+  var selected = data.selectedColumns.filter(function (col) {
+    return data.primaryKeys.indexOf(col) === -1;
+  });
+
+  chooser.open({
+    columns: nonPkColumns,
+    selected: selected,
+    onApply: function (cols) {
+      var params = new URLSearchParams(location.search);
+      params.delete("_col");
+      params.delete("_nocol");
+      params.delete("_next");
+
+      if (cols.length === nonPkColumns.length) {
+        // Check if order matches original - if so, no params needed
+        var orderMatches = cols.every(function (col, i) {
+          return col === nonPkColumns[i];
+        });
+        if (!orderMatches) {
+          cols.forEach(function (col) {
+            params.append("_col", col);
+          });
+        }
+      } else {
+        cols.forEach(function (col) {
+          params.append("_col", col);
+        });
+      }
+      var qs = params.toString();
+      location.href = qs ? "?" + qs : location.pathname;
+    },
+  });
+}
+
+function initCountAll() {
+  var button = document.querySelector(".count-all");
+  if (!button) {
+    return;
+  }
+  button.addEventListener("click", async function () {
+    var count = document.querySelector(".table-count");
+    var error = document.querySelector(".count-error");
+    button.disabled = true;
+    button.textContent = "Counting…";
+    error.textContent = "";
+    try {
+      var response = await fetch(button.dataset.countUrl + location.search, {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+        },
+      });
+      var data = await response.json();
+      if (!response.ok || !data.ok) {
+        throw new Error((data.errors || ["Count failed"]).join(" "));
+      }
+      count.textContent =
+        data.count.toLocaleString("en-US") +
+        (data.count === 1 ? " row" : " rows");
+      button.remove();
+    } catch (ex) {
+      error.textContent = ex.message || "Count failed";
+      button.disabled = false;
+      button.textContent = "count all";
+    }
+  });
+}
+
 // Ensures Table UI is initialized only after the Manager is ready.
 document.addEventListener("datasette_init", function (evt) {
   const { detail: manager } = evt;
 
+  initCountAll();
+  initializeColumnActions(manager);
+
   // Main table
   initDatasetteTable(manager);
 

datasette/stored_queries.py

@@ -0,0 +1,580 @@
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import Any
+
+from .utils import tilde_encode, urlsafe_components
+
+UNCHANGED = object()
+
+
+QUERY_OPTION_FIELDS = (
+    "hide_sql",
+    "fragment",
+    "on_success_message",
+    "on_success_message_sql",
+    "on_success_redirect",
+    "on_error_message",
+    "on_error_redirect",
+)
+
+
+@dataclass
+class StoredQuery:
+    database: str
+    name: str
+    sql: str
+    title: str | None
+    description: str | None
+    description_html: str | None
+    hide_sql: bool
+    fragment: str | None
+    parameters: list[str]
+    is_write: bool
+    is_private: bool
+    is_trusted: bool
+    source: str
+    owner_id: str | None
+    on_success_message: str | None
+    on_success_message_sql: str | None
+    on_success_redirect: str | None
+    on_error_message: str | None
+    on_error_redirect: str | None
+    private: bool | None = None
+
+
+@dataclass
+class StoredQueryPage:
+    queries: list[StoredQuery]
+    next: str | None
+    has_more: bool
+    limit: int
+
+
+def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]:
+    data = {
+        "database": query.database,
+        "name": query.name,
+        "sql": query.sql,
+        "title": query.title,
+        "description": query.description,
+        "description_html": query.description_html,
+        "hide_sql": query.hide_sql,
+        "fragment": query.fragment,
+        "parameters": list(query.parameters),
+        "is_write": query.is_write,
+        "is_private": query.is_private,
+        "is_trusted": query.is_trusted,
+        "source": query.source,
+        "owner_id": query.owner_id,
+        "on_success_message": query.on_success_message,
+        "on_success_message_sql": query.on_success_message_sql,
+        "on_success_redirect": query.on_success_redirect,
+        "on_error_message": query.on_error_message,
+        "on_error_redirect": query.on_error_redirect,
+    }
+    if query.private is not None:
+        data["private"] = query.private
+    return data
+
+
+def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]:
+    return {
+        "queries": [stored_query_to_dict(query) for query in page.queries],
+        "next": page.next,
+        "limit": page.limit,
+    }
+
+
+async def save_queries_from_config(datasette: Any) -> None:
+    # Apply configured query entries from datasette.yaml to the internal table.
+    await datasette.get_internal_database().execute_write(
+        "DELETE FROM queries WHERE source = 'config'"
+    )
+    for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items():
+        for query_name, query_config in (db_config.get("queries") or {}).items():
+            if not isinstance(query_config, dict):
+                query_config = {"sql": query_config}
+            await datasette.add_query(
+                dbname,
+                query_name,
+                query_config["sql"],
+                title=query_config.get("title"),
+                description=query_config.get("description"),
+                description_html=query_config.get("description_html"),
+                hide_sql=bool(query_config.get("hide_sql")),
+                fragment=query_config.get("fragment"),
+                parameters=query_config.get("params"),
+                is_write=bool(query_config.get("write")),
+                is_trusted=bool(query_config.get("is_trusted", True)),
+                source="config",
+                on_success_message=query_config.get("on_success_message"),
+                on_success_message_sql=query_config.get("on_success_message_sql"),
+                on_success_redirect=query_config.get("on_success_redirect"),
+                on_error_message=query_config.get("on_error_message"),
+                on_error_redirect=query_config.get("on_error_redirect"),
+            )
+
+
+def query_row_to_stored_query(
+    row: Any, private: bool | None = None
+) -> StoredQuery | None:
+    if row is None:
+        return None
+    parameters = json.loads(row["parameters"] or "[]")
+    options = json.loads(row["options"] or "{}")
+    return StoredQuery(
+        database=row["database_name"],
+        name=row["name"],
+        sql=row["sql"],
+        title=row["title"],
+        description=row["description"],
+        description_html=row["description_html"],
+        hide_sql=bool(options.get("hide_sql")),
+        fragment=options.get("fragment"),
+        parameters=parameters,
+        is_write=bool(row["is_write"]),
+        is_private=bool(row["is_private"]),
+        is_trusted=bool(row["is_trusted"]),
+        source=row["source"],
+        owner_id=row["owner_id"],
+        on_success_message=options.get("on_success_message"),
+        on_success_message_sql=options.get("on_success_message_sql"),
+        on_success_redirect=options.get("on_success_redirect"),
+        on_error_message=options.get("on_error_message"),
+        on_error_redirect=options.get("on_error_redirect"),
+        private=private,
+    )
+
+
+def query_options_json(options: dict[str, Any]) -> str:
+    options_dict = {}
+    for field in QUERY_OPTION_FIELDS:
+        value = options.get(field)
+        if field == "hide_sql":
+            if value:
+                options_dict[field] = True
+        elif value is not None:
+            options_dict[field] = value
+    return json.dumps(options_dict, sort_keys=True)
+
+
+async def add_query(
+    datasette: Any,
+    database: str,
+    name: str,
+    sql: str,
+    *,
+    title: str | None = None,
+    description: str | None = None,
+    description_html: str | None = None,
+    hide_sql: bool = False,
+    fragment: str | None = None,
+    parameters: Iterable[str] | None = None,
+    is_write: bool = False,
+    is_private: bool = False,
+    is_trusted: bool = False,
+    source: str = "plugin",
+    owner_id: str | None = None,
+    on_success_message: str | None = None,
+    on_success_message_sql: str | None = None,
+    on_success_redirect: str | None = None,
+    on_error_message: str | None = None,
+    on_error_redirect: str | None = None,
+    replace: bool = True,
+) -> None:
+    parameters_json = json.dumps(list(parameters or []))
+    options_json = query_options_json(
+        {
+            "hide_sql": hide_sql,
+            "fragment": fragment,
+            "on_success_message": on_success_message,
+            "on_success_message_sql": on_success_message_sql,
+            "on_success_redirect": on_success_redirect,
+            "on_error_message": on_error_message,
+            "on_error_redirect": on_error_redirect,
+        }
+    )
+    sql_statement = """
+        INSERT INTO queries (
+            database_name, name, sql, title, description, description_html,
+            options, parameters, is_write, is_private, is_trusted, source, owner_id
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    """
+    if replace:
+        sql_statement += """
+            ON CONFLICT(database_name, name) DO UPDATE SET
+                sql = excluded.sql,
+                title = excluded.title,
+                description = excluded.description,
+                description_html = excluded.description_html,
+                options = excluded.options,
+                parameters = excluded.parameters,
+                is_write = excluded.is_write,
+                is_private = excluded.is_private,
+                is_trusted = excluded.is_trusted,
+                source = excluded.source,
+                owner_id = excluded.owner_id,
+                updated_at = CURRENT_TIMESTAMP
+        """
+    await datasette.get_internal_database().execute_write(
+        sql_statement,
+        [
+            database,
+            name,
+            sql,
+            title,
+            description,
+            description_html,
+            options_json,
+            parameters_json,
+            int(bool(is_write)),
+            int(bool(is_private)),
+            int(bool(is_trusted)),
+            source,
+            owner_id,
+        ],
+    )
+
+
+async def update_query(
+    datasette: Any,
+    database: str,
+    name: str,
+    *,
+    sql=UNCHANGED,
+    title=UNCHANGED,
+    description=UNCHANGED,
+    description_html=UNCHANGED,
+    hide_sql=UNCHANGED,
+    fragment=UNCHANGED,
+    parameters=UNCHANGED,
+    is_write=UNCHANGED,
+    is_private=UNCHANGED,
+    is_trusted=UNCHANGED,
+    source=UNCHANGED,
+    owner_id=UNCHANGED,
+    on_success_message=UNCHANGED,
+    on_success_message_sql=UNCHANGED,
+    on_success_redirect=UNCHANGED,
+    on_error_message=UNCHANGED,
+    on_error_redirect=UNCHANGED,
+) -> None:
+    fields = {
+        "sql": sql,
+        "title": title,
+        "description": description,
+        "description_html": description_html,
+        "parameters": parameters,
+        "is_write": is_write,
+        "is_private": is_private,
+        "is_trusted": is_trusted,
+        "source": source,
+        "owner_id": owner_id,
+    }
+    option_fields = {
+        "hide_sql": hide_sql,
+        "fragment": fragment,
+        "on_success_message": on_success_message,
+        "on_success_message_sql": on_success_message_sql,
+        "on_success_redirect": on_success_redirect,
+        "on_error_message": on_error_message,
+        "on_error_redirect": on_error_redirect,
+    }
+    updates = []
+    params = []
+    for field, value in fields.items():
+        if value is UNCHANGED:
+            continue
+        if field in {"is_write", "is_private", "is_trusted"}:
+            value = int(bool(value))
+        elif field == "parameters":
+            value = json.dumps(list(value or []))
+        updates.append(f"{field} = ?")
+        params.append(value)
+    changed_options = {
+        field: value for field, value in option_fields.items() if value is not UNCHANGED
+    }
+    if changed_options:
+        rows = await datasette.get_internal_database().execute(
+            """
+            SELECT options FROM queries
+            WHERE database_name = ? AND name = ?
+            """,
+            [database, name],
+        )
+        row = rows.first()
+        options = json.loads(row["options"] or "{}") if row is not None else {}
+        for field, value in changed_options.items():
+            if field == "hide_sql":
+                if value:
+                    options[field] = True
+                else:
+                    options.pop(field, None)
+            elif value is None:
+                options.pop(field, None)
+            else:
+                options[field] = value
+        updates.append("options = ?")
+        params.append(json.dumps(options, sort_keys=True))
+    if not updates:
+        return
+    updates.append("updated_at = CURRENT_TIMESTAMP")
+    params.extend([database, name])
+    await datasette.get_internal_database().execute_write(
+        """
+        UPDATE queries
+        SET {}
+        WHERE database_name = ? AND name = ?
+        """.format(", ".join(updates)),
+        params,
+    )
+
+
+async def remove_query(
+    datasette: Any, database: str, name: str, source: str | None = None
+) -> None:
+    sql = "DELETE FROM queries WHERE database_name = ? AND name = ?"
+    params = [database, name]
+    if source is not None:
+        sql += " AND source = ?"
+        params.append(source)
+    await datasette.get_internal_database().execute_write(sql, params)
+
+
+async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None:
+    rows = await datasette.get_internal_database().execute(
+        """
+        SELECT * FROM queries
+        WHERE database_name = ? AND name = ?
+        """,
+        [database, name],
+    )
+    return query_row_to_stored_query(rows.first())
+
+
+async def count_queries(
+    datasette: Any,
+    database: str | None = None,
+    *,
+    actor: dict[str, Any] | None = None,
+    q: str | None = None,
+    is_write: bool | None = None,
+    is_private: bool | None = None,
+    is_trusted: bool | None = None,
+    source: str | None = None,
+    owner_id: str | None = None,
+) -> int:
+    allowed_sql, allowed_params = await datasette.allowed_resources_sql(
+        action="view-query",
+        actor=actor,
+        parent=database,
+    )
+    params = dict(allowed_params)
+    where_clauses = []
+    if database is not None:
+        params["query_database"] = database
+        where_clauses.append("q.database_name = :query_database")
+
+    if q:
+        where_clauses.append("""
+            (
+                q.name LIKE :query_search
+                OR q.title LIKE :query_search
+                OR q.description LIKE :query_search
+                OR q.sql LIKE :query_search
+            )
+            """)
+        params["query_search"] = f"%{q}%"
+    if is_write is not None:
+        where_clauses.append("q.is_write = :query_is_write")
+        params["query_is_write"] = int(bool(is_write))
+    if is_private is not None:
+        where_clauses.append("q.is_private = :query_is_private")
+        params["query_is_private"] = int(bool(is_private))
+    if is_trusted is not None:
+        where_clauses.append("q.is_trusted = :query_is_trusted")
+        params["query_is_trusted"] = int(bool(is_trusted))
+    if source is not None:
+        where_clauses.append("q.source = :query_source")
+        params["query_source"] = source
+    if owner_id is not None:
+        where_clauses.append("q.owner_id = :query_owner_id")
+        params["query_owner_id"] = owner_id
+
+    row = (
+        await datasette.get_internal_database().execute(
+            """
+            SELECT count(*) AS count
+            FROM queries q
+            JOIN (
+                {allowed_sql}
+            ) allowed
+              ON allowed.parent = q.database_name
+             AND allowed.child = q.name
+            WHERE {where}
+            """.format(
+                allowed_sql=allowed_sql,
+                where=" AND ".join(where_clauses) or "1 = 1",
+            ),
+            params,
+        )
+    ).first()
+    return row["count"]
+
+
+async def list_queries(
+    datasette: Any,
+    database: str | None = None,
+    *,
+    actor: dict[str, Any] | None = None,
+    limit: int = 50,
+    cursor: str | None = None,
+    q: str | None = None,
+    is_write: bool | None = None,
+    is_private: bool | None = None,
+    is_trusted: bool | None = None,
+    source: str | None = None,
+    owner_id: str | None = None,
+    include_private: bool = False,
+) -> StoredQueryPage:
+    limit = min(max(1, int(limit)), 1000)
+    allowed_sql, allowed_params = await datasette.allowed_resources_sql(
+        action="view-query",
+        actor=actor,
+        parent=database,
+        include_is_private=include_private,
+    )
+    params = dict(allowed_params)
+    params.update({"limit": limit + 1})
+    sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))"
+    where_clauses = []
+    order_by = "q.database_name, sort_key, q.name"
+    if database is not None:
+        params["query_database"] = database
+        where_clauses.append("q.database_name = :query_database")
+        order_by = "sort_key, q.name"
+
+    if cursor:
+        try:
+            components = urlsafe_components(cursor)
+        except ValueError:
+            components = []
+        if database is None and len(components) == 3:
+            where_clauses.append(f"""
+                (
+                    q.database_name > :cursor_database
+                    OR (
+                        q.database_name = :cursor_database
+                        AND (
+                            {sort_key_sql} > :cursor_sort_key
+                            OR (
+                                {sort_key_sql} = :cursor_sort_key
+                                AND q.name > :cursor_name
+                            )
+                        )
+                    )
+                )
+                """)
+            params["cursor_database"] = components[0]
+            params["cursor_sort_key"] = components[1]
+            params["cursor_name"] = components[2]
+        elif database is not None and len(components) == 2:
+            where_clauses.append(f"""
+                (
+                    {sort_key_sql} > :cursor_sort_key
+                    OR (
+                        {sort_key_sql} = :cursor_sort_key
+                        AND q.name > :cursor_name
+                    )
+                )
+                """)
+            params["cursor_sort_key"] = components[0]
+            params["cursor_name"] = components[1]
+
+    if q:
+        where_clauses.append("""
+            (
+                q.name LIKE :query_search
+                OR q.title LIKE :query_search
+                OR q.description LIKE :query_search
+                OR q.sql LIKE :query_search
+            )
+            """)
+        params["query_search"] = f"%{q}%"
+    if is_write is not None:
+        where_clauses.append("q.is_write = :query_is_write")
+        params["query_is_write"] = int(bool(is_write))
+    if is_private is not None:
+        where_clauses.append("q.is_private = :query_is_private")
+        params["query_is_private"] = int(bool(is_private))
+    if is_trusted is not None:
+        where_clauses.append("q.is_trusted = :query_is_trusted")
+        params["query_is_trusted"] = int(bool(is_trusted))
+    if source is not None:
+        where_clauses.append("q.source = :query_source")
+        params["query_source"] = source
+    if owner_id is not None:
+        where_clauses.append("q.owner_id = :query_owner_id")
+        params["query_owner_id"] = owner_id
+
+    private_select = ", allowed.is_private AS private" if include_private else ""
+    rows = list(
+        (
+            await datasette.get_internal_database().execute(
+                """
+                SELECT q.*, {sort_key_sql} AS sort_key{private_select}
+                FROM queries q
+                JOIN (
+                    {allowed_sql}
+                ) allowed
+                  ON allowed.parent = q.database_name
+                 AND allowed.child = q.name
+                WHERE {where}
+                ORDER BY {order_by}
+                LIMIT :limit
+                """.format(
+                    allowed_sql=allowed_sql,
+                    private_select=private_select,
+                    sort_key_sql=sort_key_sql,
+                    where=" AND ".join(where_clauses) or "1 = 1",
+                    order_by=order_by,
+                ),
+                params,
+            )
+        ).rows
+    )
+    has_more = len(rows) > limit
+    if has_more:
+        rows = rows[:limit]
+
+    queries = []
+    for row in rows:
+        query = query_row_to_stored_query(
+            row, private=bool(row["private"]) if include_private else None
+        )
+        assert query is not None
+        queries.append(query)
+
+    next_token = None
+    if has_more and rows:
+        last_row = rows[-1]
+        if database is None:
+            next_token = "{},{},{}".format(
+                tilde_encode(last_row["database_name"]),
+                tilde_encode(last_row["sort_key"]),
+                tilde_encode(last_row["name"]),
+            )
+        else:
+            next_token = "{},{}".format(
+                tilde_encode(last_row["sort_key"]),
+                tilde_encode(last_row["name"]),
+            )
+    return StoredQueryPage(
+        queries=queries,
+        next=next_token,
+        has_more=has_more,
+        limit=limit,
+    )

datasette/telemetry.py

@@ -0,0 +1,481 @@
+"""
+OpenTelemetry integration for Datasette.
+
+This uses `opentelemetry-api` only. Providers, exporters and sampling are
+configured by whoever runs Datasette, for example `opentelemetry-instrument`.
+"""
+
+import contextvars
+import re
+import threading
+import time
+import weakref
+from contextlib import contextmanager
+
+from opentelemetry import context as otel_context_api
+from opentelemetry import metrics as otel_metrics
+from opentelemetry import trace as otel_trace
+from opentelemetry.propagate import extract
+from opentelemetry.propagators.textmap import Getter
+from opentelemetry.trace import Link, SpanKind, Status, StatusCode, get_current_span
+
+from .telemetry_registry import (
+    DB_NAMESPACE,
+    DB_SYSTEM,
+    ERROR_TYPE,
+    HTTP_REQUEST_METHOD,
+    HTTP_RESPONSE_STATUS_CODE,
+    INTERNAL_CLIENT,
+    M_CONNECTIONS_OPEN,
+    M_OPERATION_DURATION,
+    M_QUERIES_INTERRUPTED,
+    M_QUERIES_PENDING,
+    M_THREADS_LIMIT,
+    M_THREADS_QUEUE_DEPTH,
+    M_WRITE_QUEUE_DEPTH,
+    M_WRITE_QUEUE_WAIT,
+    OPERATION,
+    SERVER_ADDRESS,
+    URL_PATH,
+    URL_SCHEME,
+    USER_AGENT_ORIGINAL,
+)
+from .version import __version__
+
+# True while code is executing within a datasette.client request. Defined
+# here rather than in app.py to avoid a circular import.
+_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False)
+
+# The semantic conventions version matching the attribute names used here.
+# 1.30.0 renamed `db.system` to `db.system.name`, so update this when
+# renaming attributes to match a newer version.
+SCHEMA_URL = "https://opentelemetry.io/schemas/1.29.0"
+
+tracer = otel_trace.get_tracer("datasette", __version__, schema_url=SCHEMA_URL)
+meter = otel_metrics.get_meter("datasette", __version__, schema_url=SCHEMA_URL)
+
+MAX_SQL_LENGTH = 2048
+
+
+def sql_attribute(sql: str) -> str:
+    "Truncate SQL text so it is safe to attach to a span as an attribute."
+    sql = sql.strip()
+    if len(sql) <= MAX_SQL_LENGTH:
+        return sql
+    return sql[:MAX_SQL_LENGTH] + "…[truncated]"
+
+
+def callback_name(fn) -> str:
+    """
+    The name recorded as `datasette.callback` for a callback-style call.
+
+    Falls back to the type name for callables such as `functools.partial`
+    that have no `__qualname__`.
+    """
+    return getattr(fn, "__qualname__", type(fn).__name__)
+
+
+def linked_root_span_kwargs(context=None):
+    """
+    Keyword arguments that start a new root span with a ``Link`` back to
+    the current span.
+
+    Use this for work that can outlive the span that caused it, such as a
+    background task or a ``block=False`` write.
+
+    Pass ``context`` to link to the span in a previously captured context
+    instead of the current one. If there is no valid span, no link is added.
+
+    Works with any tracer::
+
+        with my_tracer.start_as_current_span(
+            "myplugin.job", **linked_root_span_kwargs()
+        ):
+            ...
+    """
+    cause = get_current_span(context).get_span_context()
+    links = [Link(cause)] if cause.is_valid else []
+    return {"context": otel_context_api.Context(), "links": links}
+
+
+# Keywords that can be recorded as db.operation.name. SQL can be supplied by
+# users, so an allowlist keeps the number of distinct values small.
+DB_OPERATION_ALLOWLIST = frozenset(
+    {
+        "SELECT",
+        "INSERT",
+        "UPDATE",
+        "DELETE",
+        "CREATE",
+        "DROP",
+        "ALTER",
+        "PRAGMA",
+        "EXPLAIN",
+        "REPLACE",
+        "VACUUM",
+        "ANALYZE",
+        "WITH",
+    }
+)
+
+_LEADING_KEYWORD = re.compile(r"^\s*([A-Za-z]+)")
+
+
+def sql_operation_name(sql: str) -> str | None:
+    """
+    The statement's leading keyword if it is in the allowlist, else None.
+
+    Statements that start with a comment or "(" return None. Statements
+    starting with a CTE return `WITH`. Only call this for a single statement.
+    """
+    match = _LEADING_KEYWORD.match(sql)
+    if not match:
+        return None
+    keyword = match.group(1).upper()
+    if keyword in DB_OPERATION_ALLOWLIST:
+        return keyword
+    return None
+
+
+# --- The HTTP request span ------------------------------------------------
+
+
+class _ScopeHeadersGetter(Getter):
+    "Read W3C trace context from an ASGI scope's headers."
+
+    def get(self, carrier, key):
+        wanted = key.lower().encode("latin-1")
+        values = [v.decode("latin-1") for k, v in carrier if k.lower() == wanted]
+        return values or None
+
+    def keys(self, carrier):
+        return [k.decode("latin-1") for k, _ in carrier]
+
+
+_HEADERS_GETTER = _ScopeHeadersGetter()
+
+
+# Methods defined by RFC 9110 plus PATCH (RFC 5789). Anything else is
+# recorded as `_OTHER`, as recommended by semantic conventions.
+_KNOWN_METHODS = frozenset(
+    {"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"}
+)
+
+
+def clamp_http_method(method):
+    "The request method if it is one we recognise, else ``_OTHER``."
+    method = (method or "").upper()
+    return method if method in _KNOWN_METHODS else "_OTHER"
+
+
+def _first_header(headers, name):
+    "The first value of a header, decoded, or None."
+    for key, value in headers:
+        if key.lower() == name:
+            return value.decode("latin-1")
+    return None
+
+
+def _url_path(scope):
+    """
+    The request path, with any query string removed.
+
+    Prefers `raw_path`, which preserves encoded slashes in database and
+    table names. Some clients include the query string in `raw_path`, so
+    that is stripped as well.
+    """
+    raw_path = scope.get("raw_path")
+    if raw_path:
+        if isinstance(raw_path, bytes):
+            raw_path = raw_path.decode("latin-1")
+        return raw_path.split("?", 1)[0]
+    return scope.get("path", "")
+
+
+# The request span is passed to the router in the ASGI scope, because a
+# plugin's asgi_wrapper() middleware may have made its own span current.
+# Absent if the span is not recording.
+REQUEST_SPAN_SCOPE_KEY = "datasette.telemetry.request_span"
+
+
+def request_span(scope):
+    """
+    The recording request span for an ASGI scope, or None.
+
+    Falls back to the current span, for when Datasette is running under
+    other instrumentation.
+    """
+    span = scope.get(REQUEST_SPAN_SCOPE_KEY)
+    if span is None:
+        span = otel_trace.get_current_span()
+    return span if span.is_recording() else None
+
+
+class TelemetryMiddleware:
+    """
+    One `SpanKind.SERVER` span per HTTP request.
+
+    The span ends after the full response, including any streamed body,
+    has been sent.
+    """
+
+    def __init__(self, app):
+        self.app = app
+
+    async def __call__(self, scope, receive, send):
+        # Pass lifespan and websocket scopes straight through
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+        headers = scope.get("headers") or []
+        # Uses the global propagator, configured with OTEL_PROPAGATORS
+        context = extract(headers, getter=_HEADERS_GETTER)
+        method = clamp_http_method(scope.get("method", ""))
+        # Renamed to include the route once routing has happened
+        with tracer.start_as_current_span(
+            method, context=context, kind=SpanKind.SERVER
+        ) as span:
+            if not span.is_recording():
+                # No provider installed, or the trace was not sampled
+                await self.app(scope, receive, send)
+                return
+            span.set_attribute(HTTP_REQUEST_METHOD, method)
+            span.set_attribute(URL_PATH, _url_path(scope))
+            scheme = scope.get("scheme")
+            if scheme:
+                span.set_attribute(URL_SCHEME, scheme)
+            host = _first_header(headers, b"host")
+            if host:
+                span.set_attribute(SERVER_ADDRESS, host)
+            user_agent = _first_header(headers, b"user-agent")
+            if user_agent:
+                span.set_attribute(USER_AGENT_ORIGINAL, user_agent)
+            if _in_datasette_client.get():
+                span.set_attribute(INTERNAL_CLIENT, True)
+
+            scope = dict(scope, **{REQUEST_SPAN_SCOPE_KEY: span})
+
+            # Some responses are sent without a Response object, so the
+            # status is captured by wrapping send()
+            status_holder = {}
+
+            async def wrapped_send(message):
+                if (
+                    message["type"] == "http.response.start"
+                    and "status" not in status_holder
+                ):
+                    status_holder["status"] = message["status"]
+                await send(message)
+
+            escaped = False
+            try:
+                await self.app(scope, receive, wrapped_send)
+            except BaseException as exception:
+                # Includes asyncio.CancelledError when a client disconnects
+                escaped = True
+                span.set_attribute(ERROR_TYPE, type(exception).__name__)
+                span.set_status(Status(StatusCode.ERROR, str(exception)))
+                raise
+            finally:
+                status = status_holder.get("status")
+                if status is not None:
+                    span.set_attribute(HTTP_RESPONSE_STATUS_CODE, status)
+                    # 4xx responses are not errors for a server span. If an
+                    # exception escaped, keep its class name as error.type.
+                    if status >= 500 and not escaped:
+                        span.set_status(Status(StatusCode.ERROR))
+                        span.set_attribute(ERROR_TYPE, str(status))
+
+
+# --- Metrics --------------------------------------------------------------
+
+
+def _duration_attributes(database_name, operation):
+    return {
+        DB_SYSTEM: "sqlite",
+        DB_NAMESPACE: database_name,
+        OPERATION: operation,
+    }
+
+
+# Instruments use plain text descriptions. The registry entries have longer
+# reStructuredText descriptions for the documentation.
+
+sql_operation_duration = meter.create_histogram(
+    M_OPERATION_DURATION,
+    unit=M_OPERATION_DURATION.unit,
+    description="Duration of a SQL operation issued by Datasette",
+    explicit_bucket_boundaries_advisory=M_OPERATION_DURATION.buckets,
+)
+
+write_queue_wait = meter.create_histogram(
+    M_WRITE_QUEUE_WAIT,
+    unit=M_WRITE_QUEUE_WAIT.unit,
+    description=(
+        "Time a write spent queued behind the single write thread for its database"
+    ),
+    explicit_bucket_boundaries_advisory=M_WRITE_QUEUE_WAIT.buckets,
+)
+
+queries_interrupted = meter.create_counter(
+    M_QUERIES_INTERRUPTED,
+    unit=M_QUERIES_INTERRUPTED.unit,
+    description="Queries cancelled for exceeding sql_time_limit_ms",
+)
+
+
+@contextmanager
+def record_operation_duration(database_name, operation):
+    """
+    Record `db.client.operation.duration` for one SQL operation.
+
+    Sets `error.type` to the exception class on failure. For a `block=False`
+    write this measures the time taken to enqueue the write.
+    """
+    attributes = _duration_attributes(database_name, operation)
+    started = time.perf_counter()
+    try:
+        yield
+    except BaseException as exception:
+        attributes[ERROR_TYPE] = type(exception).__qualname__
+        raise
+    finally:
+        sql_operation_duration.record(time.perf_counter() - started, attributes)
+
+
+def record_write_queue_wait(database_name, waited_ns):
+    write_queue_wait.record(waited_ns / 1e9, {DB_NAMESPACE: database_name})
+
+
+def record_query_interrupted(database_name):
+    queries_interrupted.add(1, {DB_NAMESPACE: database_name})
+
+
+# Live Datasette instances reported by the gauges below. The lock is needed
+# because gauge callbacks run on the SDK's collection thread.
+#
+# The pool gauges do not identify which instance they came from, so they
+# are only meaningful for a process running a single Datasette instance.
+_live_datasettes = weakref.WeakSet()
+_live_datasettes_lock = threading.Lock()
+
+
+def register_datasette(ds):
+    "Start reporting pool/queue gauges for this Datasette instance."
+    with _live_datasettes_lock:
+        _live_datasettes.add(ds)
+
+
+def unregister_datasette(ds):
+    "Stop reporting gauges for an instance that has been closed."
+    with _live_datasettes_lock:
+        _live_datasettes.discard(ds)
+
+
+def _live_instances():
+    with _live_datasettes_lock:
+        return list(_live_datasettes)
+
+
+def _databases_of(ds):
+    "Every Database attached to an instance, including the internal database."
+    databases = list(ds.databases.values())
+    internal = getattr(ds, "_internal_database", None)
+    if internal is not None:
+        databases.append(internal)
+    return databases
+
+
+def observe_sql_thread_limit(options=None):
+    "Size of the shared read-query thread pool (the num_sql_threads setting)."
+    for ds in _live_instances():
+        if ds.executor is None:
+            # num_sql_threads=0 - queries run on the event loop, no pool.
+            continue
+        yield otel_metrics.Observation(ds.setting("num_sql_threads"), {})
+
+
+def observe_sql_thread_queue_depth(options=None):
+    """
+    Read queries waiting for a free thread in the shared pool.
+
+    `_work_queue` is a private attribute of ThreadPoolExecutor, so this
+    reports nothing if it is missing.
+    """
+    for ds in _live_instances():
+        if ds.executor is None:
+            continue
+        work_queue = getattr(ds.executor, "_work_queue", None)
+        if work_queue is None:
+            continue
+        yield otel_metrics.Observation(work_queue.qsize(), {})
+
+
+def observe_pending_queries(options=None):
+    """
+    Read queries submitted to the pool and not yet finished, per database.
+
+    Reads `len()` without `_pending_execute_futures_lock` to avoid blocking
+    queries.
+    """
+    for ds in _live_instances():
+        for db in _databases_of(ds):
+            yield otel_metrics.Observation(
+                len(db._pending_execute_futures), {DB_NAMESPACE: db.name}
+            )
+
+
+def observe_write_queue_depth(options=None):
+    "Writes queued behind the single write thread, per database."
+    for ds in _live_instances():
+        for db in _databases_of(ds):
+            write_queue = db._write_queue
+            if write_queue is None:
+                # No write has ever been queued for this database.
+                continue
+            yield otel_metrics.Observation(write_queue.qsize(), {DB_NAMESPACE: db.name})
+
+
+def observe_open_connections(options=None):
+    "Open SQLite connections tracked for closing, per database."
+    for ds in _live_instances():
+        for db in _databases_of(ds):
+            yield otel_metrics.Observation(
+                len(db._all_connections), {DB_NAMESPACE: db.name}
+            )
+
+
+sql_thread_limit_gauge = meter.create_observable_gauge(
+    M_THREADS_LIMIT,
+    callbacks=[observe_sql_thread_limit],
+    unit=M_THREADS_LIMIT.unit,
+    description="Maximum concurrent read queries (the num_sql_threads setting)",
+)
+
+sql_thread_queue_depth_gauge = meter.create_observable_gauge(
+    M_THREADS_QUEUE_DEPTH,
+    callbacks=[observe_sql_thread_queue_depth],
+    unit=M_THREADS_QUEUE_DEPTH.unit,
+    description="Read queries waiting for a free thread in the shared SQL pool",
+)
+
+pending_queries_gauge = meter.create_observable_gauge(
+    M_QUERIES_PENDING,
+    callbacks=[observe_pending_queries],
+    unit=M_QUERIES_PENDING.unit,
+    description="Read queries submitted to the pool and not yet complete",
+)
+
+write_queue_depth_gauge = meter.create_observable_gauge(
+    M_WRITE_QUEUE_DEPTH,
+    callbacks=[observe_write_queue_depth],
+    unit=M_WRITE_QUEUE_DEPTH.unit,
+    description="Writes queued behind a database's single write thread",
+)
+
+open_connections_gauge = meter.create_observable_gauge(
+    M_CONNECTIONS_OPEN,
+    callbacks=[observe_open_connections],
+    unit=M_CONNECTIONS_OPEN.unit,
+    description="Open SQLite connections tracked for closing",
+)

datasette/telemetry_registry.py

@@ -0,0 +1,502 @@
+"""
+Every span, metric and attribute that Datasette emits.
+
+These entries are used by the instrumentation code, by `docs/telemetry_doc.py`
+to generate the documentation, and by `tests/test_telemetry_registry.py` to
+check that the emitted telemetry matches the registry.
+"""
+
+from opentelemetry.trace import SpanKind
+
+
+class Attribute(str):
+    """
+    A span attribute key, carrying its own documentation.
+
+    Subclasses `str` so it can be handed straight to `set_attribute()`.
+
+    Part of Datasette's public plugin API - plugins declare their own
+    telemetry registries with these classes. See the "Telemetry for plugin
+    authors" documentation.
+    """
+
+    __slots__ = ("description", "optional", "values")
+
+    def __new__(cls, name, description, optional=False, values=None):
+        self = super().__new__(cls, name)
+        self.description = description
+        self.optional = optional
+        # The allowed values for this attribute, or None to allow any value
+        self.values = frozenset(values) if values is not None else None
+        return self
+
+    def __reduce__(self):
+        # Copies and pickles become a plain str, since __new__ requires the
+        # extra arguments. ConsoleMetricExporter deepcopies attribute keys.
+        return (str, (str(self),))
+
+    def __repr__(self):
+        return f"Attribute({str(self)!r})"
+
+
+class SpanName(str):
+    """A span name, carrying its documentation and the attributes it may set.
+
+    Part of Datasette's public plugin API, like `Attribute`.
+    """
+
+    __slots__ = ("attributes", "description", "dynamic", "kind", "prefix")
+
+    def __new__(
+        cls,
+        name,
+        description,
+        attributes=(),
+        prefix=False,
+        dynamic=False,
+        kind=SpanKind.INTERNAL,
+    ):
+        self = super().__new__(cls, name)
+        self.description = description
+        self.attributes = tuple(attributes)
+        # Match emitted names that start with this prefix, for names with a
+        # variable suffix such as SpanName("chat ", ..., prefix=True)
+        self.prefix = prefix
+        # The emitted name is built at runtime, so `span_for()` matches it by
+        # span kind. The entry's string is a template for the documentation.
+        self.dynamic = dynamic
+        self.kind = kind
+        return self
+
+    def __reduce__(self):
+        # See Attribute.__reduce__.
+        return (str, (str(self),))
+
+    def __repr__(self):
+        return f"SpanName({str(self)!r})"
+
+
+class MetricName(str):
+    "A metric name, carrying its instrument kind, unit and attributes."
+
+    __slots__ = ("attributes", "buckets", "description", "kind", "unit")
+
+    def __new__(cls, name, kind, unit, description, attributes=(), buckets=None):
+        self = super().__new__(cls, name)
+        self.kind = kind
+        self.unit = unit
+        self.description = description
+        self.attributes = tuple(attributes)
+        # Explicit bucket boundaries, for histograms only
+        self.buckets = tuple(buckets) if buckets is not None else None
+        return self
+
+    def __reduce__(self):
+        # See Attribute.__reduce__.
+        return (str, (str(self),))
+
+    def __repr__(self):
+        return f"MetricName({str(self)!r})"
+
+
+COUNTER = "Counter"
+UPDOWN_COUNTER = "UpDownCounter"
+HISTOGRAM = "Histogram"
+GAUGE = "Observable gauge"
+
+
+# --- Attributes -----------------------------------------------------------
+
+HTTP_REQUEST_METHOD = Attribute(
+    "http.request.method",
+    "The HTTP request method. Methods outside the nine defined by RFC 9110 "
+    "and RFC 5789 are recorded as ``_OTHER``.",
+)
+HTTP_RESPONSE_STATUS_CODE = Attribute(
+    "http.response.status_code",
+    "The HTTP response status code. Omitted if no response was started.",
+    optional=True,
+)
+HTTP_ROUTE = Attribute(
+    "http.route",
+    "The regular expression for the matched route, for example "
+    "``/(?P<database>[^\\/\\.]+)/(?P<table>[^\\/\\.]+)(\\.(?P<format>\\w+))?$`` "
+    "for a table page. Use this attribute to group requests by route. "
+    "Omitted when no route matches.",
+    optional=True,
+)
+URL_PATH = Attribute(
+    "url.path",
+    "The URL path, excluding the query string.",
+)
+URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.")
+SERVER_ADDRESS = Attribute(
+    "server.address",
+    "The ``Host`` header, including any ``:port`` suffix. This value is "
+    "supplied by the client.",
+    optional=True,
+)
+USER_AGENT_ORIGINAL = Attribute(
+    "user_agent.original",
+    "The ``User-Agent`` header, verbatim. Omitted if the client sent none.",
+    optional=True,
+)
+INTERNAL_CLIENT = Attribute(
+    "datasette.internal_client",
+    "``True`` for requests made through ``datasette.client``. Calls made "
+    "inside another request produce a nested ``SERVER`` span. Filter on "
+    "this attribute to exclude internal requests from request counts. "
+    "Omitted for requests received over the network.",
+    optional=True,
+)
+ERROR_TYPE = Attribute(
+    "error.type",
+    "The exception class name for a failed operation. On HTTP spans, also "
+    "set to the status code as a string for 5xx responses. A 4xx response "
+    "alone does not set this attribute or an error status.",
+    optional=True,
+)
+
+DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.")
+DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.")
+OPERATION = Attribute(
+    "datasette.operation",
+    "Whether the operation was a read or a write.",
+    values={"read", "write"},
+)
+DB_QUERY_TEXT = Attribute(
+    "db.query.text",
+    "The SQL, truncated to 2048 characters. Bound parameter values are not "
+    "recorded. For callback methods, ``datasette.callback`` is recorded instead.",
+    optional=True,
+)
+CALLBACK = Attribute(
+    "datasette.callback",
+    "The qualified name of the Python callable passed to ``execute_fn()``, "
+    "``execute_write_fn()`` or ``execute_isolated_fn()``, for example "
+    "``TableInsertView.post.<locals>.insert_or_upsert_rows``. Set instead of "
+    "``db.query.text``. Lambdas appear as ``<lambda>``; use a named function "
+    "for a more descriptive span.",
+    optional=True,
+)
+DB_OPERATION_NAME = Attribute(
+    "db.operation.name",
+    "The statement's leading keyword, such as ``SELECT``, ``INSERT`` or "
+    "``CREATE``, if it matches the supported allowlist. Statements beginning "
+    "with a common table expression report ``WITH``. Omitted for unrecognized "
+    "keywords and ``execute_write_script()``.",
+    optional=True,
+)
+PARAM_COUNT = Attribute(
+    "datasette.param_count",
+    "Number of bound parameters. Recorded instead of the values themselves.",
+    optional=True,
+)
+PARAM_SETS = Attribute(
+    "datasette.param_sets",
+    "Number of parameter sets consumed by ``execute_write_many()``. "
+    "The parameter values are not recorded.",
+    optional=True,
+)
+TIME_LIMIT_MS = Attribute(
+    "datasette.time_limit_ms",
+    "Time limit applied to the read query, in milliseconds: "
+    ":ref:`setting_sql_time_limit_ms` or a shorter ``custom_time_limit``.",
+    optional=True,
+)
+ROWS_RETURNED = Attribute(
+    "datasette.rows_returned",
+    "Number of rows returned by a successful read query.",
+    optional=True,
+)
+TRUNCATED = Attribute(
+    "datasette.truncated",
+    "True if the result was cut short by :ref:`setting_max_returned_rows`.",
+    optional=True,
+)
+INTERRUPTED = Attribute(
+    "datasette.interrupted",
+    "True if the query exceeded its time limit. The span status is set to "
+    "``ERROR`` unless the caller used a ``custom_time_limit`` shorter than "
+    ":ref:`setting_sql_time_limit_ms`, in which case the status is left unset.",
+    optional=True,
+)
+SQL_ERROR_SUPPRESSED = Attribute(
+    "datasette.sql_error_suppressed",
+    "True for a non-timeout SQL error with ``log_sql_errors=False``. The "
+    "exception is still raised, but the span status is left unset.",
+    optional=True,
+)
+EXECUTESCRIPT = Attribute(
+    "datasette.executescript",
+    "True for ``execute_write_script()``, which runs multiple statements.",
+    optional=True,
+)
+EXECUTEMANY = Attribute(
+    "datasette.executemany",
+    "True for ``execute_write_many()``, which runs one statement against many "
+    "parameter sets.",
+    optional=True,
+)
+ISOLATED_CONNECTION = Attribute(
+    "datasette.isolated_connection",
+    "True if the write ran on its own connection rather than the shared write "
+    "connection.",
+)
+TRANSACTION = Attribute(
+    "datasette.transaction",
+    "False for statements such as ``VACUUM`` that cannot run inside a transaction.",
+)
+
+
+# --- Spans ----------------------------------------------------------------
+
+HTTP_REQUEST = SpanName(
+    "{http.request.method} {http.route}",
+    "One span per HTTP request, containing spans from plugin middleware and "
+    "database operations. Named for the HTTP method and matched route, or "
+    "just the method if no route matches. Incoming ``traceparent`` headers "
+    "are extracted using the global propagator to continue the caller's "
+    "trace. Incoming ``baggage`` is not propagated into plugin or downstream "
+    "context in this release. Set ``OTEL_PROPAGATORS=none`` to disable "
+    "extraction. For public instances, strip trace context headers at your "
+    "proxy if callers should not supply trace context.",
+    (
+        HTTP_REQUEST_METHOD,
+        HTTP_ROUTE,
+        URL_PATH,
+        URL_SCHEME,
+        SERVER_ADDRESS,
+        USER_AGENT_ORIGINAL,
+        HTTP_RESPONSE_STATUS_CODE,
+        ERROR_TYPE,
+        INTERNAL_CLIENT,
+    ),
+    dynamic=True,
+    kind=SpanKind.SERVER,
+)
+
+DB_QUERY = SpanName(
+    "db.query",
+    "A SQL operation, including time spent queued for a worker thread. For "
+    "``block=False`` writes, the span ends after the write is queued. "
+    "Callback methods record ``datasette.callback`` in place of ``db.query.text``.",
+    (
+        DB_SYSTEM,
+        DB_NAMESPACE,
+        DB_QUERY_TEXT,
+        CALLBACK,
+        DB_OPERATION_NAME,
+        PARAM_COUNT,
+        PARAM_SETS,
+        TIME_LIMIT_MS,
+        ROWS_RETURNED,
+        TRUNCATED,
+        INTERRUPTED,
+        SQL_ERROR_SUPPRESSED,
+        EXECUTESCRIPT,
+        EXECUTEMANY,
+    ),
+    kind=SpanKind.CLIENT,
+)
+
+DB_QUERY_EXECUTE = SpanName(
+    "db.query.execute",
+    "The read executing inside a SQL worker thread. Child of ``db.query``; the "
+    "gap between the two is time spent waiting for a thread.",
+)
+
+DB_WRITE_QUEUE_WAIT = SpanName(
+    "db.write.queue_wait",
+    "Time a write spent waiting in its database's write queue. For "
+    "``block=True``, this is a child of ``db.query``. For ``block=False``, "
+    "it is a root span linked to the span that queued the write, since the "
+    "write can outlive that request.",
+)
+
+DB_WRITE_EXECUTE = SpanName(
+    "db.write.execute",
+    "The write executing on the write thread. For ``block=True``, this is "
+    "a child of ``db.query``. For ``block=False``, it is a root span linked "
+    "to the span that queued the write.",
+    (ISOLATED_CONNECTION, TRANSACTION),
+)
+
+STARTUP = SpanName(
+    "datasette.startup",
+    "Startup work performed by ``invoke_startup()``, including registration "
+    "hooks, schema catalog updates, saved queries, column type configuration "
+    "and the ``startup`` hook. Runs during instance startup, either before "
+    "serving requests or as part of the first request.",
+)
+
+SPANS = (
+    HTTP_REQUEST,
+    DB_QUERY,
+    DB_QUERY_EXECUTE,
+    DB_WRITE_QUEUE_WAIT,
+    DB_WRITE_EXECUTE,
+    STARTUP,
+)
+
+
+def span_for(emitted_name, kind=None, spans=None):
+    """
+    Resolve an emitted span name to its registry entry, or None.
+
+    Exact matches take precedence over `prefix=True` entries, which take
+    precedence over `dynamic=True` entries matched by `kind`.
+
+    `spans` defaults to Datasette's own registry.
+    """
+    if spans is None:
+        spans = SPANS
+    for span in spans:
+        if span.dynamic:
+            continue
+        if emitted_name == span:
+            return span
+    for span in spans:
+        if span.prefix and emitted_name.startswith(span):
+            return span
+    if kind is not None:
+        for span in spans:
+            if span.dynamic and span.kind == kind:
+                return span
+    return None
+
+
+def metric_for(emitted_name, metrics=None):
+    """
+    Resolve an emitted metric name to its registry entry, or None.
+
+    `metrics` defaults to Datasette's own registry.
+    """
+    if metrics is None:
+        metrics = METRICS
+    for metric in metrics:
+        if emitted_name == metric:
+            return metric
+    return None
+
+
+def attribute_allowed(entry, emitted_key):
+    """
+    Whether `emitted_key` is a registered attribute of `entry`.
+
+    `entry` is a `SpanName` or a `MetricName` - both carry `.attributes`.
+    """
+    if entry is None:
+        return False
+    return emitted_key in entry.attributes
+
+
+def attribute_value_allowed(entry, emitted_key, value):
+    """
+    Whether `value` is permitted for `emitted_key` on `entry` (a `SpanName`
+    or a `MetricName`).
+
+    Any value is allowed if the attribute does not declare `values=`.
+    """
+    if entry is None:
+        return False
+    for attribute in entry.attributes:
+        if attribute == emitted_key:
+            return attribute.values is None or value in attribute.values
+    return False
+
+
+# --- Metrics --------------------------------------------------------------
+
+# Bucket boundaries in seconds for every duration histogram. OpenTelemetry's
+# defaults are designed for milliseconds and would put almost every SQLite
+# query in the first bucket. These are the semantic conventions' recommended
+# boundaries for db.client.operation.duration, plus 0.0001 and 0.0005 for
+# fast in-process SQLite queries.
+DURATION_BUCKETS = (0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10)
+
+M_OPERATION_DURATION = MetricName(
+    "db.client.operation.duration",
+    HISTOGRAM,
+    "s",
+    "Duration of a SQL operation, including callback-based calls such as "
+    "``execute_fn()``. For ``block=False`` writes, measures enqueue time.",
+    (DB_SYSTEM, DB_NAMESPACE, OPERATION, ERROR_TYPE),
+    buckets=DURATION_BUCKETS,
+)
+
+M_WRITE_QUEUE_WAIT = MetricName(
+    "datasette.write.queue_wait",
+    HISTOGRAM,
+    "s",
+    "Time each write waited in its database's write queue.",
+    (DB_NAMESPACE,),
+    buckets=DURATION_BUCKETS,
+)
+
+M_QUERIES_INTERRUPTED = MetricName(
+    "datasette.sql.queries.interrupted",
+    COUNTER,
+    "{query}",
+    "Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. A "
+    "rising rate can indicate that queries need optimization or a higher "
+    "time limit. Caller-selected timeouts shorter than this limit, such as "
+    "those used for facet suggestion, are excluded.",
+    (DB_NAMESPACE,),
+)
+
+M_THREADS_LIMIT = MetricName(
+    "datasette.sql.threads.limit",
+    GAUGE,
+    "{thread}",
+    "Maximum concurrent read queries, configured by "
+    ":ref:`setting_num_sql_threads`. Not reported when ``num_sql_threads`` "
+    "is ``0``.",
+)
+
+M_THREADS_QUEUE_DEPTH = MetricName(
+    "datasette.sql.threads.queue_depth",
+    GAUGE,
+    "{query}",
+    "Read queries waiting for a free SQL thread. Sustained values above "
+    "zero indicate a saturated read pool.",
+)
+
+M_QUERIES_PENDING = MetricName(
+    "datasette.sql.queries.pending",
+    GAUGE,
+    "{query}",
+    "Read queries submitted to the pool and not yet complete. Sum across "
+    "databases and compare with ``datasette.sql.threads.limit`` to assess "
+    "pool usage.",
+    (DB_NAMESPACE,),
+)
+
+M_WRITE_QUEUE_DEPTH = MetricName(
+    "datasette.write.queue_depth",
+    GAUGE,
+    "{write}",
+    "Writes waiting for a database's single write thread. Increasing "
+    "``num_sql_threads`` does not increase write concurrency. Not reported for "
+    "databases that have never been written to.",
+    (DB_NAMESPACE,),
+)
+
+M_CONNECTIONS_OPEN = MetricName(
+    "datasette.connections.open",
+    GAUGE,
+    "{connection}",
+    "Open SQLite connections managed by Datasette.",
+    (DB_NAMESPACE,),
+)
+
+METRICS = (
+    M_OPERATION_DURATION,
+    M_WRITE_QUEUE_WAIT,
+    M_QUERIES_INTERRUPTED,
+    M_THREADS_LIMIT,
+    M_THREADS_QUEUE_DEPTH,
+    M_QUERIES_PENDING,
+    M_WRITE_QUEUE_DEPTH,
+    M_CONNECTIONS_OPEN,
+)

datasette/telemetry_testing.py

@@ -0,0 +1,427 @@
+"""
+Pytest helpers for testing OpenTelemetry instrumentation - Datasette's own
+and any plugin's. Part of Datasette's public plugin API; see the "Telemetry
+for plugin authors" documentation.
+
+Usage from a plugin's ``conftest.py``::
+
+    from datasette.telemetry_testing import (  # noqa: F401
+        MetricsCollector,
+        otel_metrics,
+        otel_meter_provider,
+        otel_provider,
+        otel_spans,
+    )
+
+Tests can then use the ``otel_spans`` and ``otel_metrics`` fixtures. The
+OpenTelemetry SDK is imported lazily, and the fixtures skip if it is not
+installed.
+"""
+
+import subprocess
+import sys
+
+import pytest
+
+from .telemetry_registry import (
+    attribute_allowed,
+    attribute_value_allowed,
+    metric_for,
+    span_for,
+)
+
+_span_exporter = None
+_metric_reader = None
+
+
+def install_span_exporter():
+    """
+    Install a TracerProvider + InMemorySpanExporter once per process and
+    return the exporter, or None when the SDK is not installed.
+
+    Uses `SimpleSpanProcessor` so spans are exported as soon as they end.
+    """
+    global _span_exporter
+    if _span_exporter is not None:
+        return _span_exporter
+    try:
+        from opentelemetry import trace as otel_trace
+        from opentelemetry.sdk.trace import TracerProvider
+        from opentelemetry.sdk.trace.export import SimpleSpanProcessor
+        from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
+            InMemorySpanExporter,
+        )
+    except ImportError:
+        return None
+    exporter = InMemorySpanExporter()
+    provider = TracerProvider()
+    provider.add_span_processor(SimpleSpanProcessor(exporter))
+    otel_trace.set_tracer_provider(provider)
+    # set_tracer_provider() is ignored if a provider was already installed,
+    # in which case the fixtures skip
+    if otel_trace.get_tracer_provider() is not provider:
+        return None
+    _span_exporter = exporter
+    return exporter
+
+
+def install_metric_reader():
+    """
+    Install a MeterProvider + InMemoryMetricReader once per process and
+    return the reader, or None when the SDK is not installed.
+
+    Uses delta temporality for counters and histograms, so each collection
+    only reports measurements since the previous one.
+    """
+    global _metric_reader
+    if _metric_reader is not None:
+        return _metric_reader
+    try:
+        from opentelemetry import metrics as otel_metrics_api
+        from opentelemetry.sdk.metrics import Counter, Histogram, MeterProvider
+        from opentelemetry.sdk.metrics.export import (
+            AggregationTemporality,
+            InMemoryMetricReader,
+        )
+    except ImportError:
+        return None
+    reader = InMemoryMetricReader(
+        preferred_temporality={
+            Counter: AggregationTemporality.DELTA,
+            Histogram: AggregationTemporality.DELTA,
+        }
+    )
+    provider = MeterProvider(metric_readers=[reader])
+    otel_metrics_api.set_meter_provider(provider)
+    if otel_metrics_api.get_meter_provider() is not provider:
+        return None
+    _metric_reader = reader
+    return reader
+
+
+@pytest.fixture(scope="session", autouse=True)
+def otel_provider():
+    "Install the span exporter once per test session, before any spans are created."
+    install_span_exporter()
+
+
+@pytest.fixture(scope="session", autouse=True)
+def otel_meter_provider():
+    "Install the metric reader once per test session."
+    install_metric_reader()
+
+
+@pytest.fixture(autouse=True)
+def otel_reset():
+    "Clear recorded spans and drain collected metrics after every test."
+    yield
+    if _span_exporter is not None:
+        _span_exporter.clear()
+    if _metric_reader is not None:
+        _metric_reader.get_metrics_data()
+
+
+@pytest.fixture
+def otel_spans():
+    """
+    The in-memory span exporter, cleared before the test. Call
+    `.get_finished_spans()` to retrieve spans.
+    """
+    pytest.importorskip("opentelemetry.sdk")
+    exporter = install_span_exporter()
+    if exporter is None:
+        pytest.skip("OpenTelemetry SDK provider was not installed")
+    exporter.clear()
+    yield exporter
+
+
+class MetricsCollector:
+    """
+    Wraps an `InMemoryMetricReader`.
+
+    `collect()` runs a collection cycle and stores a snapshot, which
+    `points()` and `point()` then query.
+    """
+
+    def __init__(self, reader):
+        self.reader = reader
+        self.snapshot = {}
+        # (instrumentation scope name, sdk Metric) pairs from the last collect()
+        self.collected = []
+
+    def collect(self):
+        self.snapshot = {}
+        self.collected = []
+        data = self.reader.get_metrics_data()
+        if data is None:
+            return self.snapshot
+        for resource_metrics in data.resource_metrics:
+            for scope_metrics in resource_metrics.scope_metrics:
+                scope_name = scope_metrics.scope.name if scope_metrics.scope else None
+                for metric in scope_metrics.metrics:
+                    self.snapshot.setdefault(metric.name, []).extend(
+                        metric.data.data_points
+                    )
+                    self.collected.append((scope_name, metric))
+        return self.snapshot
+
+    def points(self, name, attributes=None):
+        "Data points for `name` whose attributes are a superset of `attributes`."
+        found = []
+        for point in self.snapshot.get(name, []):
+            point_attributes = dict(point.attributes or {})
+            if all(point_attributes.get(k) == v for k, v in (attributes or {}).items()):
+                found.append(point)
+        return found
+
+    def point(self, name, attributes=None):
+        "The single matching data point, asserting there is exactly one."
+        found = self.points(name, attributes)
+        assert len(found) == 1, (
+            f"expected exactly one {name} point matching {attributes}, "
+            f"got {len(found)}: {found}"
+        )
+        return found[0]
+
+
+@pytest.fixture
+def otel_metrics():
+    "A `MetricsCollector`, drained before the test so counts start from zero."
+    pytest.importorskip("opentelemetry.sdk")
+    reader = install_metric_reader()
+    if reader is None:
+        pytest.skip("OpenTelemetry SDK meter provider was not installed")
+    reader.get_metrics_data()
+    yield MetricsCollector(reader)
+
+
+def _scoped(finished_spans, scope_name):
+    if scope_name is None:
+        return list(finished_spans)
+    return [
+        span
+        for span in finished_spans
+        if span.instrumentation_scope and span.instrumentation_scope.name == scope_name
+    ]
+
+
+def assert_spans_conform(registry_spans, finished_spans, scope_name=None):
+    """
+    Assert every finished span is registered in `registry_spans`, sets only
+    registered attributes and uses allowed attribute values.
+
+    Pass `scope_name` to only check spans from that instrumentation scope.
+    """
+    problems = []
+    for span in _scoped(finished_spans, scope_name):
+        entry = span_for(str(span.name), kind=span.kind, spans=registry_spans)
+        if entry is None:
+            problems.append(f"unregistered span: {span.name!r}")
+            continue
+        for key, value in (span.attributes or {}).items():
+            if not attribute_allowed(entry, str(key)):
+                problems.append(f"{span.name}: unregistered attribute {key!r}")
+            elif not attribute_value_allowed(entry, str(key), value):
+                problems.append(
+                    f"{span.name}: {key}={value!r} not in the declared enum"
+                )
+    assert not problems, "\n".join(problems)
+
+
+def assert_spans_covered(registry_spans, finished_spans, scope_name=None):
+    """
+    Assert every entry in `registry_spans` was emitted at least once, with
+    each of its attributes that is not `optional=True`.
+    """
+    spans = _scoped(finished_spans, scope_name)
+    seen_attributes = {}
+    for span in spans:
+        entry = span_for(str(span.name), kind=span.kind, spans=registry_spans)
+        if entry is not None:
+            seen = seen_attributes.setdefault(str(entry), set())
+            seen.update(str(key) for key in (span.attributes or {}))
+    problems = []
+    for entry in registry_spans:
+        if str(entry) not in seen_attributes:
+            problems.append(f"registered span never emitted: {entry!r}")
+            continue
+        required = {
+            str(attribute) for attribute in entry.attributes if not attribute.optional
+        }
+        missing = required - seen_attributes[str(entry)]
+        if missing:
+            problems.append(
+                f"{entry}: registered attributes never emitted: {sorted(missing)}"
+            )
+    assert not problems, "\n".join(problems)
+
+
+# Registry instrument kinds mapped to the SDK data type collected for them.
+# Both counter kinds collect as Sum, distinguished by is_monotonic.
+_KIND_TO_DATA_TYPE = {
+    "Counter": "Sum",
+    "UpDownCounter": "Sum",
+    "Histogram": "Histogram",
+    "Observable gauge": "Gauge",
+}
+_KIND_IS_MONOTONIC = {"Counter": True, "UpDownCounter": False}
+
+
+def _scoped_metrics(collector, scope_name):
+    for scope, metric in collector.collected:
+        if scope_name is None or scope == scope_name:
+            yield metric
+
+
+def assert_metrics_conform(registry_metrics, collector, scope_name=None):
+    """
+    Assert every metric in the collector's last `collect()` is registered in
+    `registry_metrics` with a matching instrument kind and unit, sets only
+    registered attributes and uses allowed attribute values.
+
+    Pass `scope_name` to only check metrics from that instrumentation scope.
+    """
+    problems = set()
+    for metric in _scoped_metrics(collector, scope_name):
+        entry = metric_for(metric.name, metrics=registry_metrics)
+        if entry is None:
+            problems.add(f"unregistered metric: {metric.name!r}")
+            continue
+        expected_data_type = _KIND_TO_DATA_TYPE.get(entry.kind)
+        actual_data_type = type(metric.data).__name__
+        if expected_data_type is not None and actual_data_type != expected_data_type:
+            problems.add(
+                f"{metric.name}: registry declares {entry.kind}, "
+                f"SDK collected {actual_data_type}"
+            )
+        expected_monotonic = _KIND_IS_MONOTONIC.get(entry.kind)
+        actual_monotonic = getattr(metric.data, "is_monotonic", None)
+        if (
+            expected_monotonic is not None
+            and actual_monotonic is not None
+            and actual_monotonic != expected_monotonic
+        ):
+            problems.add(
+                f"{metric.name}: registry declares {entry.kind}, but the "
+                f"collected Sum is_monotonic={actual_monotonic}"
+            )
+        if (metric.unit or "") != (entry.unit or ""):
+            problems.add(
+                f"{metric.name}: instrument unit {metric.unit!r} != "
+                f"registry unit {entry.unit!r}"
+            )
+        for point in metric.data.data_points:
+            for key, value in dict(point.attributes or {}).items():
+                if not attribute_allowed(entry, str(key)):
+                    problems.add(f"{metric.name}: unregistered attribute {key!r}")
+                elif not attribute_value_allowed(entry, str(key), value):
+                    problems.add(
+                        f"{metric.name}: {key}={value!r} not in the declared enum"
+                    )
+    assert not problems, "\n".join(sorted(problems))
+
+
+def assert_metrics_covered(registry_metrics, collector, scope_name=None):
+    """
+    Assert every entry in `registry_metrics` was collected at least once,
+    with each of its attributes that is not `optional=True`.
+
+    Call `collect()` once after the workload and before this check.
+    """
+    seen_attributes = {}
+    for metric in _scoped_metrics(collector, scope_name):
+        entry = metric_for(metric.name, metrics=registry_metrics)
+        if entry is None:
+            continue
+        seen = seen_attributes.setdefault(str(entry), set())
+        for point in metric.data.data_points:
+            seen.update(str(key) for key in dict(point.attributes or {}))
+    problems = []
+    for entry in registry_metrics:
+        if str(entry) not in seen_attributes:
+            problems.append(f"registered metric never collected: {entry!r}")
+            continue
+        required = {
+            str(attribute) for attribute in entry.attributes if not attribute.optional
+        }
+        missing = required - seen_attributes[str(entry)]
+        if missing:
+            problems.append(
+                f"{entry}: registered attributes never collected: {sorted(missing)}"
+            )
+    assert not problems, "\n".join(problems)
+
+
+def assert_no_forbidden_values(
+    forbidden, finished_spans=None, collector=None, scope_name=None
+):
+    """
+    Assert that none of the `forbidden` strings appear anywhere in the
+    emitted telemetry: span names, span attribute values, span event names
+    and attributes, span status descriptions, or metric point attributes.
+
+    Use fake private values such as tokens or email addresses in your test
+    workload, then check that they were not recorded:
+
+        FORBIDDEN = {"secret-token-123", "alice@example.com"}
+        run_workload_using_those_values()
+        assert_no_forbidden_values(
+            FORBIDDEN,
+            finished_spans=otel_spans.get_finished_spans(),
+            collector=otel_metrics,
+        )
+
+    Matches substrings of each value's string form. Empty strings in
+    `forbidden` are ignored. Leave `scope_name` unset to also check
+    Datasette's own telemetry.
+    """
+    needles = [needle for needle in forbidden if needle]
+    leaks = set()
+
+    def check(value, where):
+        text = str(value)
+        for needle in needles:
+            if needle in text:
+                leaks.add(f"{where} contains {needle!r}")
+
+    if finished_spans is not None:
+        for span in _scoped(finished_spans, scope_name):
+            check(span.name, f"span name {str(span.name)!r}")
+            for key, value in (span.attributes or {}).items():
+                check(value, f"{span.name} attribute {key}")
+            for event in span.events or ():
+                check(event.name, f"{span.name} event name")
+                for key, value in (event.attributes or {}).items():
+                    check(value, f"{span.name} event {event.name} attribute {key}")
+            if span.status is not None and span.status.description:
+                check(span.status.description, f"{span.name} status description")
+    if collector is not None:
+        for metric in _scoped_metrics(collector, scope_name):
+            for point in metric.data.data_points:
+                for key, value in dict(point.attributes or {}).items():
+                    check(value, f"metric {metric.name} attribute {key}")
+    assert not leaks, "forbidden values leaked into telemetry:\n" + "\n".join(
+        sorted(leaks)
+    )
+
+
+def assert_package_never_imports_sdk(*module_names):
+    """
+    Import the named modules in a fresh interpreter and assert none of them
+    imported `opentelemetry.sdk`.
+
+    Run the test that calls this early in your suite: on macOS with CPython
+    3.13, starting a subprocess from a process with many threads can crash.
+    """
+    imports = "; ".join(f"import {name}" for name in module_names)
+    code = (
+        f"import sys; {imports}; "
+        "print([m for m in sys.modules if m.startswith('opentelemetry.sdk')])"
+    )
+    result = subprocess.run(
+        [sys.executable, "-c", code], capture_output=True, text=True, check=True
+    )
+    assert result.stdout.strip() == "[]", (
+        f"importing {module_names} pulled in the OpenTelemetry SDK: "
+        f"{result.stdout.strip()}"
+    )

datasette/template_contexts.py

@@ -0,0 +1,40 @@
+"""
+Index of the documented template contexts for Datasette's core HTML pages.
+
+This module deliberately contains no documentation strings of its own -
+the documentation lives next to the code it describes:
+
+- Every page renders a Context dataclass defined in its view module
+  (DatabaseContext, QueryContext in views/database.py, TableContext in
+  views/table.py, RowContext in views/row.py). Fields added by view code
+  carry ``help`` metadata; fields declared with from_extra() take their
+  documentation from the description on the matching Extra class in
+  views/table_extras.py.
+- The keys render_template() adds to every page are documented in
+  TEMPLATE_BASE_CONTEXT in datasette/app.py, next to the code that adds
+  them.
+
+The contract tests in tests/test_template_context.py assert that the real
+rendered context for each page exactly matches what is documented, and
+docs/template_context_doc.py generates docs/template_context.rst from the
+same classes.
+"""
+
+from datasette.app import TEMPLATE_BASE_CONTEXT
+from datasette.views.database import DatabaseContext, QueryContext
+from datasette.views.row import RowContext
+from datasette.views.table import TableContext
+
+PAGES = {
+    "database": DatabaseContext,
+    "query": QueryContext,
+    "table": TableContext,
+    "row": RowContext,
+}
+
+
+def documented_context_keys(page_name):
+    "Set of every documented key for the named page, including base context keys"
+    return set(TEMPLATE_BASE_CONTEXT) | {
+        f.name for f in PAGES[page_name].documented_fields()
+    }

datasette/templates/_action_menu.html

@@ -1,7 +1,7 @@
 {% if action_links %}
 <div class="page-action-menu">
 <details class="actions-menu-links details-menu">
-    <summary>
+    <summary aria-haspopup="menu" aria-expanded="false">
         <div class="icon-text">
             <svg class="icon" aria-labelledby="actions-menu-links-title" role="img" style="color: #fff" xmlns="http://www.w3.org/2000/svg" width="28" height="28" viewBox="0 0 28 28" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
                 <title id="actions-menu-links-title">{{ action_title }}</title>
@@ -13,12 +13,20 @@
     </summary>
     <div class="dropdown-menu">
         <div class="hook"></div>
-        <ul>
+        <ul role="menu">
             {% for link in action_links %}
-            <li><a href="{{ link.href }}">{{ link.label }}
+            <li role="none">
+            {% if link.get("type") == "button" %}
+                <button type="button" class="button-as-link action-menu-button" role="menuitem" tabindex="-1"{% for name, value in (link.get("attrs") or {}).items() %} {{ name }}="{{ value }}"{% endfor %}>{{ link.label }}
                 {% if link.description %}
-                <p class="dropdown-description">{{ link.description }}</p>
+                    <span class="dropdown-description">{{ link.description }}</span>
+                {% endif %}</button>
+            {% else %}
+                <a href="{{ link.href }}" role="menuitem" tabindex="-1">{{ link.label }}
+                {% if link.description %}
+                    <span class="dropdown-description">{{ link.description }}</span>
                 {% endif %}</a>
+            {% endif %}
             </li>
             {% endfor %}
         </ul>

datasette/templates/_close_open_menus.html

@@ -13,4 +13,50 @@ document.body.addEventListener('click', (ev) => {
         (details) => details.open && details != detailsClickedWithin
     ).forEach(details => details.open = false);
 });
+
+/* Sync aria-expanded and add keyboard navigation for details-menu elements */
+document.querySelectorAll('details.details-menu').forEach(function(details) {
+    var summary = details.querySelector('summary');
+    details.addEventListener('toggle', function() {
+        if (summary) {
+            summary.setAttribute('aria-expanded', details.open ? 'true' : 'false');
+        }
+        if (details.open) {
+            /* Focus first menu item when menu opens */
+            var firstItem = details.querySelector('[role="menuitem"]');
+            if (firstItem) { firstItem.focus(); }
+        }
+    });
+});
+
+document.body.addEventListener('keydown', function(ev) {
+    /* Keyboard navigation for open details-menu elements */
+    var openDetails = Array.from(document.querySelectorAll('details.details-menu[open]'));
+    if (!openDetails.length) { return; }
+
+    if (ev.key === 'Escape') {
+        openDetails.forEach(function(details) {
+            details.open = false;
+            var summary = details.querySelector('summary');
+            if (summary) { summary.focus(); }
+        });
+        return;
+    }
+
+    if (ev.key === 'ArrowDown' || ev.key === 'ArrowUp') {
+        var focused = document.activeElement;
+        openDetails.forEach(function(details) {
+            var items = Array.from(details.querySelectorAll('[role="menuitem"]'));
+            if (!items.length) { return; }
+            var idx = items.indexOf(focused);
+            if (idx === -1) { return; }
+            ev.preventDefault();
+            if (ev.key === 'ArrowDown') {
+                items[(idx + 1) % items.length].focus();
+            } else {
+                items[(idx - 1 + items.length) % items.length].focus();
+            }
+        });
+    }
+});
 </script>

datasette/templates/_codemirror.html

@@ -1,5 +1,5 @@
-<script src="{{ base_url }}-/static/sql-formatter-2.3.3.min.js" defer></script>
+<script src="{{ static('sql-formatter-2.3.3.min.js') }}" defer></script>
-<script src="{{ base_url }}-/static/cm-editor-6.0.1.bundle.js"></script>
+<script src="{{ static('cm-editor-6.0.1.bundle.js') }}"></script>
 <style>
   .cm-editor {
     resize: both;

datasette/templates/_execute_write_analysis_scripts.html

@@ -0,0 +1,111 @@
+<script>
+window.datasetteSqlAnalysis = (() => {
+  if (
+    window.datasetteSqlAnalysis &&
+    window.datasetteSqlAnalysis.renderAnalysis
+  ) {
+    return window.datasetteSqlAnalysis;
+  }
+
+  function appendCodeCell(row, value, emptyText) {
+    const cell = document.createElement("td");
+    if (value) {
+      const code = document.createElement("code");
+      code.textContent = value;
+      cell.appendChild(code);
+    } else if (emptyText) {
+      appendNotApplicable(cell);
+    }
+    row.appendChild(cell);
+  }
+
+  function appendNotApplicable(cell) {
+    const notApplicable = document.createElement("span");
+    notApplicable.className = "execute-write-analysis-na";
+    notApplicable.textContent = "n/a";
+    cell.appendChild(notApplicable);
+  }
+
+  function renderAnalysis(section, data) {
+    if (!section) {
+      return;
+    }
+    section.replaceChildren();
+    if (data.has_sql === false) {
+      section.hidden = true;
+      return;
+    }
+    section.hidden = false;
+
+    const heading = document.createElement("h2");
+    heading.textContent = "Query operations";
+    section.appendChild(heading);
+
+    if (data.analysis_error) {
+      const error = document.createElement("p");
+      error.className = "message-error";
+      error.textContent = data.analysis_error;
+      section.appendChild(error);
+      return;
+    }
+
+    const rows = data.analysis_rows || [];
+    if (!rows.length) {
+      const empty = document.createElement("p");
+      empty.textContent =
+        "Analysis will show each affected table and required permission.";
+      section.appendChild(empty);
+      return;
+    }
+
+    const wrapper = document.createElement("div");
+    wrapper.className = "table-wrapper";
+    const table = document.createElement("table");
+    table.className = "execute-write-analysis";
+    const thead = document.createElement("thead");
+    const headerRow = document.createElement("tr");
+    [
+      "Operation",
+      "Database",
+      "Table",
+      "Required permission",
+      "Allowed",
+    ].forEach((label) => {
+      const th = document.createElement("th");
+      th.scope = "col";
+      th.textContent = label;
+      headerRow.appendChild(th);
+    });
+    thead.appendChild(headerRow);
+    table.appendChild(thead);
+
+    const tbody = document.createElement("tbody");
+    rows.forEach((analysisRow) => {
+      const row = document.createElement("tr");
+      appendCodeCell(row, analysisRow.operation);
+      appendCodeCell(row, analysisRow.database);
+      appendCodeCell(row, analysisRow.table);
+      appendCodeCell(row, analysisRow.required_permission, "n/a");
+
+      const allowedCell = document.createElement("td");
+      if (analysisRow.allowed !== null && analysisRow.allowed !== undefined) {
+        const allowed = document.createElement("span");
+        allowed.className = analysisRow.allowed
+          ? "execute-write-analysis-allowed"
+          : "execute-write-analysis-denied";
+        allowed.textContent = analysisRow.allowed ? "yes" : "no";
+        allowedCell.appendChild(allowed);
+      } else {
+        appendNotApplicable(allowedCell);
+      }
+      row.appendChild(allowedCell);
+      tbody.appendChild(row);
+    });
+    table.appendChild(tbody);
+    wrapper.appendChild(table);
+    section.appendChild(wrapper);
+  }
+
+  return { renderAnalysis };
+})();
+</script>

datasette/templates/_execute_write_analysis_styles.html

@@ -0,0 +1,41 @@
+<style>
+.execute-write-analysis {
+    border-collapse: collapse;
+    font-size: 0.9rem;
+    margin: 0.25rem 0 1rem;
+    min-width: 44rem;
+}
+.execute-write-analysis th,
+.execute-write-analysis td {
+    border-bottom: 1px solid #d7dde5;
+    padding: 0.45rem 0.7rem;
+    text-align: left;
+    vertical-align: top;
+}
+.execute-write-analysis th {
+    background-color: #edf6fb;
+    border-top: 1px solid #d7dde5;
+    color: #39445a;
+    font-weight: 700;
+}
+.execute-write-analysis tbody tr:nth-child(even) {
+    background-color: rgba(39, 104, 144, 0.05);
+}
+.execute-write-analysis code {
+    background: transparent;
+    font-size: 0.9em;
+    white-space: nowrap;
+}
+.execute-write-analysis-allowed {
+    color: #267a3e;
+    font-weight: 700;
+}
+.execute-write-analysis-denied {
+    color: #b00020;
+    font-weight: 700;
+}
+.execute-write-analysis-na {
+    color: #687386;
+    font-style: italic;
+}
+</style>

datasette/templates/_facet_results.html

@@ -12,9 +12,9 @@
             <ul class="tight-bullets">
                 {% for facet_value in facet_info.results %}
                     {% if not facet_value.selected %}
-                        <li><a href="{{ facet_value.toggle_url }}" data-facet-value="{{ facet_value.value }}">{{ (facet_value.label | string()) or "-" }}</a> {{ "{:,}".format(facet_value.count) }}</li>
+                        <li><a href="{{ facet_value.toggle_url }}" data-facet-value="{{ facet_value.value }}">{{ (facet_value.label | string()) or "-" }}</a> <span class="facet-count">{{ "{:,}".format(facet_value.count) }}</span></li>
                     {% else %}
-                        <li>{{ facet_value.label or "-" }} &middot; {{ "{:,}".format(facet_value.count) }} <a href="{{ facet_value.toggle_url }}" class="cross">&#x2716;</a></li>
+                        <li>{{ facet_value.label or "-" }} &middot; <span class="facet-count">{{ "{:,}".format(facet_value.count) }}</span> <a href="{{ facet_value.toggle_url }}" class="cross">&#x2716;</a></li>
                     {% endif %}
                 {% endfor %}
                 {% if facet_info.truncated %}

datasette/templates/_permission_ui_styles.html

@@ -6,8 +6,20 @@
     padding: 1.5em;
     margin-bottom: 2em;
 }
+.permission-form form {
+    max-width: 60rem;
+}
+.permission-form-grid {
+    display: grid;
+    gap: 1.5rem;
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+}
+.permission-form-result {
+    margin-top: 1rem;
+    max-width: 60rem;
+}
 .form-section {
-    margin-bottom: 1em;
+    margin-bottom: 1.25em;
 }
 .form-section label {
     display: block;
@@ -15,22 +27,51 @@
     font-weight: bold;
 }
 .form-section input[type="text"],
-.form-section select {
+.form-section input[type="number"],
-    width: 100%;
+.form-section select,
-    max-width: 500px;
+.permission-textarea {
-    padding: 0.5em;
+    background-color: #fff;
+    border: 1px solid #aaa;
+    border-radius: 4px;
     box-sizing: border-box;
-    border: 1px solid #ccc;
+    box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.08);
-    border-radius: 3px;
+    color: #222;
+    font-family: inherit;
+    font-size: 1rem;
+    line-height: 1.4;
+    max-width: none;
+    width: 100%;
+}
+.form-section input[type="text"] {
+    height: 3rem;
+    padding: 0.6rem 0.75rem;
+}
+.form-section input[type="number"] {
+    height: 3rem;
+    max-width: 7rem;
+    padding: 0.6rem 0.75rem;
+}
+.form-section select {
+    height: 3rem;
+    padding: 0.6rem 0.75rem;
+}
+.permission-textarea {
+    font-family: monospace;
+    min-height: 12rem;
+    padding: 0.75rem;
+    resize: vertical;
 }
 .form-section input[type="text"]:focus,
-.form-section select:focus {
+.form-section input[type="number"]:focus,
-    outline: 2px solid #0066cc;
+.form-section select:focus,
+.permission-textarea:focus {
     border-color: #0066cc;
+    box-shadow: 0 0 0 3px rgba(0, 102, 204, 0.18);
+    outline: none;
 }
 .form-section small {
     display: block;
-    margin-top: 0.3em;
+    margin-top: 0.45em;
     color: #666;
 }
 .form-actions {
@@ -142,4 +183,9 @@
     text-align: center;
     color: #666;
 }
+@media only screen and (max-width: 576px) {
+    .permission-form-grid {
+        grid-template-columns: minmax(0, 1fr);
+    }
+}
 </style>

datasette/templates/_permissions_debug_tabs.html

@@ -44,10 +44,10 @@
 </style>
 
 <nav class="permissions-debug-tabs">
-    <a href="{{ urls.path('-/permissions') }}" {% if current_tab == "permissions" %}class="active"{% endif %}>Playground</a>
+    <a href="{{ urls.path('-/check') }}{{ query_string }}" {% if current_tab == "check" %}class="active"{% endif %}>Explain</a>
-    <a href="{{ urls.path('-/check') }}{{ query_string }}" {% if current_tab == "check" %}class="active"{% endif %}>Check</a>
+    <a href="{{ urls.path('-/allowed') }}{{ query_string }}" {% if current_tab == "allowed" %}class="active"{% endif %}>Access map</a>
-    <a href="{{ urls.path('-/allowed') }}{{ query_string }}" {% if current_tab == "allowed" %}class="active"{% endif %}>Allowed</a>
+    <a href="{{ urls.path('-/rules') }}{{ query_string }}" {% if current_tab == "rules" %}class="active"{% endif %}>Rule explorer</a>
-    <a href="{{ urls.path('-/rules') }}{{ query_string }}" {% if current_tab == "rules" %}class="active"{% endif %}>Rules</a>
+    <a href="{{ urls.path('-/permissions') }}" {% if current_tab == "permissions" %}class="active"{% endif %}>Activity</a>
     <a href="{{ urls.path('-/actions') }}" {% if current_tab == "actions" %}class="active"{% endif %}>Actions</a>
     <a href="{{ urls.path('-/allow-debug') }}" {% if current_tab == "allow_debug" %}class="active"{% endif %}>Allow debug</a>
 </nav>

datasette/templates/_query_form_styles.html

@@ -0,0 +1,138 @@
+<style>
+.query-create-page {
+    max-width: 64rem;
+}
+.query-create-form {
+    --query-create-label-width: clamp(7rem, 18vw, 10rem);
+    --query-create-column-gap: 0.8rem;
+    --query-create-control-width: minmax(16rem, 1fr);
+}
+.query-create-fields {
+    margin: 0 0 0.85rem;
+    max-width: 52rem;
+}
+.query-create-field {
+    align-items: start;
+    column-gap: var(--query-create-column-gap);
+    display: grid;
+    grid-template-columns: var(--query-create-label-width) var(--query-create-control-width);
+    margin: 0 0 0.65rem;
+}
+.query-create-field label {
+    padding-top: 0.55rem;
+    width: auto;
+}
+.query-create-field input[type=text],
+.query-create-field textarea {
+    box-sizing: border-box;
+    width: 100%;
+}
+form.sql .query-create-field textarea {
+    width: 100%;
+}
+.query-create-url-control {
+    align-items: center;
+    box-sizing: border-box;
+    display: grid;
+    gap: 0.35rem;
+    grid-template-columns: max-content minmax(12rem, 1fr);
+    width: 100%;
+}
+.query-create-url-prefix {
+    color: #4f5b6d;
+    font-family: var(--font-monospace, monospace);
+    white-space: nowrap;
+}
+.query-create-url-control input[type=text] {
+    border: 1px solid #ccc;
+    border-radius: 3px;
+}
+.query-create-url-static {
+    color: #39445a;
+    font-family: var(--font-monospace, monospace);
+    word-break: break-all;
+}
+.query-create-field textarea {
+    border: 1px solid #ccc;
+    border-radius: 3px;
+    display: block;
+    font-family: Helvetica, sans-serif;
+    font-size: 1em;
+    min-height: 5rem;
+    padding: 9px 4px;
+    resize: vertical;
+}
+form.sql .query-create-sql {
+    column-gap: var(--query-create-column-gap);
+    display: grid;
+    grid-template-columns: var(--query-create-label-width) var(--query-create-control-width);
+    margin: 0.9rem 0 0.75rem;
+    max-width: 52rem;
+}
+.query-create-sql .cm-editor,
+form.sql .query-create-sql textarea#sql-editor {
+    grid-column: 2;
+    width: 100%;
+}
+.query-create-options {
+    align-items: center;
+    display: flex;
+    flex-wrap: wrap;
+    gap: 0.8rem 1.4rem;
+    margin: 0 0 0.9rem calc(var(--query-create-label-width) + var(--query-create-column-gap));
+    max-width: calc(52rem - var(--query-create-label-width) - var(--query-create-column-gap));
+}
+.query-create-options label {
+    align-items: center;
+    display: inline-flex;
+    gap: 0.35rem;
+    width: auto;
+}
+.query-create-options input[type=checkbox] {
+    margin: 0;
+}
+.query-create-option-note,
+.query-create-analysis-note {
+    color: #4f5b6d;
+    flex-basis: 100%;
+    font-size: 0.82rem;
+}
+.query-create-option-note {
+    margin: -0.45rem 0 0;
+}
+.query-create-analysis-note {
+    margin: 0;
+}
+.query-create-analysis {
+    margin-top: 0.8rem;
+}
+.query-create-submit {
+    margin-left: calc(var(--query-create-label-width) + var(--query-create-column-gap));
+    margin-bottom: 0.9rem;
+    margin-top: 1rem;
+}
+@media (max-width: 560px) {
+    .query-create-form {
+        --query-create-label-width: 1fr;
+        --query-create-column-gap: 0;
+    }
+    .query-create-field {
+        grid-template-columns: 1fr;
+        row-gap: 0.25rem;
+    }
+    .query-create-field label {
+        padding-top: 0;
+    }
+    form.sql .query-create-sql {
+        grid-template-columns: 1fr;
+    }
+    .query-create-sql .cm-editor,
+    form.sql .query-create-sql textarea#sql-editor {
+        grid-column: 1;
+    }
+    .query-create-options,
+    .query-create-submit {
+        margin-left: 0;
+    }
+}
+</style>

datasette/templates/_query_results.html

@@ -0,0 +1,20 @@
+{% if display_rows %}
+<div class="table-wrapper"><table class="rows-and-columns">
+    <thead>
+        <tr>
+            {% for column in columns %}<th class="col-{{ column|to_css_class }}" scope="col">{{ column }}</th>{% endfor %}
+        </tr>
+    </thead>
+    <tbody>
+    {% for row in display_rows %}
+        <tr>
+            {% for column, td in zip(columns, row) %}
+                <td class="col-{{ column|to_css_class }}">{{ td }}</td>
+            {% endfor %}
+        </tr>
+    {% endfor %}
+    </tbody>
+</table></div>
+{% elif show_zero_results %}
+    <p class="zero-results">0 results</p>
+{% endif %}

datasette/templates/_sql_parameter_scripts.html

@@ -0,0 +1,307 @@
+<script>
+window.datasetteSqlParameters = (() => {
+  if (
+    window.datasetteSqlParameters &&
+    window.datasetteSqlParameters.setupSqlParameterRefresh
+  ) {
+    return window.datasetteSqlParameters;
+  }
+
+  function currentSql(form) {
+    if (window.editor) {
+      return window.editor.state.doc.toString();
+    }
+    const sqlInput = form.querySelector("textarea#sql-editor, input[name=sql]");
+    return sqlInput ? sqlInput.value : "";
+  }
+
+  function controlState(control) {
+    return {
+      value: control.value,
+      expanded: control.tagName.toLowerCase() === "textarea",
+    };
+  }
+
+  function syncParameterState(manager) {
+    manager.parameterState = new Map();
+    manager.section
+      .querySelectorAll("[data-parameter-control]")
+      .forEach((control) => {
+        manager.parameterState.set(
+          control.dataset.parameterName,
+          controlState(control)
+        );
+      });
+  }
+
+  function createControl(parameter, id, state, namePrefix) {
+    const control = document.createElement(state.expanded ? "textarea" : "input");
+    control.id = id;
+    control.name = `${namePrefix || ""}${parameter}`;
+    control.value = state.value;
+    control.setAttribute("data-parameter-control", "");
+    control.dataset.parameterName = parameter;
+    if (state.expanded) {
+      control.rows = 5;
+    } else {
+      control.type = "text";
+    }
+    return control;
+  }
+
+  function replaceParameterControl(
+    manager,
+    control,
+    button,
+    expand,
+    value,
+    selectionStart
+  ) {
+    const parameter = control.dataset.parameterName;
+    const replacement = createControl(
+      parameter,
+      control.id,
+      {
+        value: value === undefined ? control.value : value,
+        expanded: expand,
+      },
+      manager.namePrefix
+    );
+    button.textContent = expand ? "Collapse" : "Expand";
+    button.setAttribute("aria-expanded", expand ? "true" : "false");
+    control.replaceWith(replacement);
+    replacement.focus();
+    if (selectionStart !== undefined && replacement.setSelectionRange) {
+      replacement.setSelectionRange(selectionStart, selectionStart);
+    }
+    manager.parameterState.set(parameter, controlState(replacement));
+  }
+
+  function renderParameters(manager, parameters) {
+    syncParameterState(manager);
+    const previousState = manager.parameterState;
+    const nextState = new Map();
+    manager.section.replaceChildren();
+    if (!parameters.length) {
+      manager.parameterState = nextState;
+      return;
+    }
+
+    const heading = document.createElement("h2");
+    heading.textContent = "Parameters";
+    manager.section.appendChild(heading);
+
+    parameters.forEach((parameter, index) => {
+      const id = `qp${index + 1}`;
+      const state = previousState.get(parameter) || {
+        value: "",
+        expanded: false,
+      };
+      if (!manager.allowExpand) {
+        state.expanded = false;
+      }
+      nextState.set(parameter, state);
+
+      const row = document.createElement("p");
+      row.className = "sql-parameter-row";
+
+      const label = document.createElement("label");
+      label.htmlFor = id;
+      label.textContent = parameter;
+
+      const control = createControl(parameter, id, state, manager.namePrefix);
+
+      row.append(label, control);
+      if (manager.allowExpand) {
+        const button = document.createElement("button");
+        button.type = "button";
+        button.className = "sql-parameter-toggle";
+        button.setAttribute("data-parameter-toggle", "");
+        button.setAttribute("aria-controls", id);
+        button.setAttribute("aria-expanded", state.expanded ? "true" : "false");
+        button.textContent = state.expanded ? "Collapse" : "Expand";
+        row.append(" ", button);
+      }
+      manager.section.appendChild(row);
+    });
+
+    manager.parameterState = nextState;
+  }
+
+  function bindParameterControls(manager) {
+    manager.form.addEventListener("input", (event) => {
+      const control = event.target;
+      if (!control.matches || !control.matches("[data-parameter-control]")) {
+        return;
+      }
+      manager.parameterState.set(
+        control.dataset.parameterName,
+        controlState(control)
+      );
+    });
+
+    if (!manager.allowExpand) {
+      return;
+    }
+
+    manager.form.addEventListener("click", (event) => {
+      const button = event.target.closest
+        ? event.target.closest("[data-parameter-toggle]")
+        : null;
+      if (!button || !manager.form.contains(button)) {
+        return;
+      }
+      const control = document.getElementById(button.getAttribute("aria-controls"));
+      if (!control) {
+        return;
+      }
+      const expanded = control.tagName.toLowerCase() === "textarea";
+      replaceParameterControl(manager, control, button, !expanded);
+    });
+
+    manager.form.addEventListener("paste", (event) => {
+      const control = event.target;
+      if (
+        !(control instanceof HTMLInputElement) ||
+        !control.matches("[data-parameter-control]")
+      ) {
+        return;
+      }
+      const pasted = event.clipboardData ? event.clipboardData.getData("text") : "";
+      if (!/[\r\n]/.test(pasted)) {
+        return;
+      }
+      const button = document.querySelector(
+        `[data-parameter-toggle][aria-controls="${control.id}"]`
+      );
+      if (!button) {
+        return;
+      }
+      event.preventDefault();
+      const selectionStart = control.selectionStart ?? control.value.length;
+      const selectionEnd = control.selectionEnd ?? selectionStart;
+      const value =
+        control.value.slice(0, selectionStart) +
+        pasted +
+        control.value.slice(selectionEnd);
+      replaceParameterControl(
+        manager,
+        control,
+        button,
+        true,
+        value,
+        selectionStart + pasted.length
+      );
+    });
+  }
+
+  function bindEditorChanges(form, callback) {
+    const editorElement = form.querySelector(".cm-content");
+    if (editorElement) {
+      editorElement.addEventListener("input", callback);
+    }
+    if (!window.editor) {
+      const sqlInput = form.querySelector("textarea#sql-editor");
+      if (sqlInput) {
+        sqlInput.addEventListener("input", callback);
+      }
+      return;
+    }
+    if (!window.editor.datasetteSqlParameterCallbacks) {
+      const editor = window.editor;
+      const originalDispatch = editor.dispatch.bind(editor);
+      editor.datasetteSqlParameterCallbacks = [];
+      editor.dispatch = (...transactions) => {
+        const before = editor.state.doc.toString();
+        originalDispatch(...transactions);
+        if (editor.state.doc.toString() !== before) {
+          editor.datasetteSqlParameterCallbacks.forEach((listener) => listener());
+        }
+      };
+    }
+    window.editor.datasetteSqlParameterCallbacks.push(callback);
+  }
+
+  function setupSqlParameterRefresh(options) {
+    const form =
+      options.form || document.querySelector("form.sql.core[data-parameters-url]");
+    if (!form) {
+      return null;
+    }
+    const shouldRenderParameters = options.renderParameters !== false;
+    const section =
+      options.section || form.querySelector("[data-sql-parameters-section]");
+    if (shouldRenderParameters && !section) {
+      return null;
+    }
+    const manager = {
+      form,
+      section,
+      allowExpand:
+        options.allowExpand === undefined
+          ? section
+            ? section.dataset.allowExpand === "1"
+            : false
+          : options.allowExpand,
+      namePrefix: section ? section.dataset.parameterNamePrefix || "" : "",
+      parameterState: new Map(),
+    };
+    if (section) {
+      bindParameterControls(manager);
+      syncParameterState(manager);
+    }
+
+    const url = options.url || form.dataset.parametersUrl;
+    let refreshTimer = null;
+    let refreshSequence = 0;
+
+    async function refreshParameters() {
+      if (!url) {
+        return;
+      }
+      const sequence = ++refreshSequence;
+      try {
+        const requestUrl = new URL(url, window.location.href);
+        requestUrl.searchParams.set("sql", currentSql(form));
+        const response = await fetch(requestUrl, {
+          headers: { accept: "application/json" },
+        });
+        const data = await response.json();
+        if (sequence !== refreshSequence) {
+          return;
+        }
+        if (!response.ok) {
+          throw new Error((data.errors || [response.statusText]).join("; "));
+        }
+        if (shouldRenderParameters) {
+          renderParameters(manager, data.parameters || []);
+        }
+        if (options.onData) {
+          options.onData(data, manager);
+        }
+      } catch (error) {
+        if (sequence !== refreshSequence) {
+          return;
+        }
+        if (options.onError) {
+          options.onError(error, manager);
+        }
+      }
+    }
+
+    function scheduleRefresh() {
+      clearTimeout(refreshTimer);
+      refreshTimer = setTimeout(refreshParameters, options.debounceMs || 350);
+    }
+
+    bindEditorChanges(form, scheduleRefresh);
+    return {
+      currentSql: () => currentSql(form),
+      refreshParameters,
+      renderParameters: (parameters) => renderParameters(manager, parameters),
+    };
+  }
+
+  return { setupSqlParameterRefresh };
+})();
+</script>

datasette/templates/_sql_parameter_styles.html

@@ -0,0 +1,58 @@
+<style>
+form.sql .sql-editor {
+    max-width: 52rem;
+}
+form.sql .sql-editor textarea#sql-editor {
+    width: 100%;
+}
+form.sql .sql-parameters-section {
+    max-width: 52rem;
+}
+form.sql .sql-parameter-row {
+    align-items: start;
+    column-gap: 0.6rem;
+    display: grid;
+    grid-template-columns: minmax(8rem, 11rem) minmax(16rem, 1fr) auto;
+    margin: 0 0 0.65rem;
+    max-width: 52rem;
+}
+form.sql .sql-parameter-row label {
+    overflow-wrap: anywhere;
+    padding-top: 0.55rem;
+    width: auto;
+}
+form.sql .sql-parameter-row input[data-parameter-control],
+form.sql .sql-parameter-row textarea[data-parameter-control] {
+    box-sizing: border-box;
+    width: 100%;
+}
+form.sql .sql-parameter-row textarea[data-parameter-control] {
+    border: 1px solid #ccc;
+    border-radius: 3px;
+    display: block;
+    font-family: Helvetica, sans-serif;
+    font-size: 1em;
+    min-height: 7rem;
+    padding: 9px 4px;
+}
+form.sql.core button.sql-parameter-toggle[type=button] {
+    font-size: 0.72rem;
+    height: 1.8rem;
+    line-height: 1;
+    margin: 0.25rem 0 0;
+    padding: 0.25rem 0.45rem;
+}
+@media (max-width: 480px) {
+    form.sql .sql-parameter-row {
+        grid-template-columns: 1fr;
+        row-gap: 0.25rem;
+    }
+    form.sql .sql-parameter-row label {
+        padding-top: 0;
+    }
+    form.sql.core button.sql-parameter-toggle[type=button] {
+        justify-self: start;
+        margin-top: 0;
+    }
+}
+</style>

datasette/templates/_sql_parameters.html

@@ -0,0 +1,10 @@
+{% set sql_parameter_name_prefix = sql_parameter_name_prefix|default("") %}
+<div id="{{ sql_parameters_section_id|default("sql-parameters-section") }}" class="sql-parameters-section" data-sql-parameters-section{% if sql_parameter_name_prefix %} data-parameter-name-prefix="{{ sql_parameter_name_prefix }}"{% endif %}{% if sql_parameters_allow_expand|default(false) %} data-allow-expand="1"{% endif %}>
+    {% if parameter_names %}
+        <h2>Parameters</h2>
+        {% for parameter in parameter_names %}
+            {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %}
+            <p class="sql-parameter-row"><label for="{{ parameter_id }}">{{ parameter }}</label> <input type="text" id="{{ parameter_id }}" name="{{ sql_parameter_name_prefix }}{{ parameter }}" value="{{ parameter_values.get(parameter, "") }}" data-parameter-control data-parameter-name="{{ parameter }}">{% if sql_parameters_allow_expand|default(false) %} <button type="button" class="sql-parameter-toggle" data-parameter-toggle aria-controls="{{ parameter_id }}" aria-expanded="false">Expand</button>{% endif %}</p>
+        {% endfor %}
+    {% endif %}
+</div>

datasette/templates/_table.html

@@ -1,12 +1,12 @@
 <!-- above-table-panel is a hook node for plugins to attach to . Displays even if no data available -->
 <div class="above-table-panel"> </div>
-{% if display_rows %}
+{% if display_columns %}
 <div class="table-wrapper">
     <table class="rows-and-columns">
         <thead>
             <tr>
                 {% for column in display_columns %}
-                    <th {% if column.description %}data-column-description="{{ column.description }}" {% endif %}class="col-{{ column.name|to_css_class }}" scope="col" data-column="{{ column.name }}" data-column-type="{{ column.type.lower() }}" data-column-not-null="{{ column.notnull }}" data-is-pk="{% if column.is_pk %}1{% else %}0{% endif %}">
+                    <th {% if column.description %}data-column-description="{{ column.description }}" {% endif %}class="col-{{ column.name|to_css_class }}" scope="col" data-column="{{ column.name }}" data-column-type="{{ column.type.lower() }}" data-column-not-null="{{ column.notnull }}" data-is-pk="{% if column.is_pk %}1{% else %}0{% endif %}"{% if column.is_special_link_column %} data-is-link-column="1"{% endif %}>
                         {% if not column.sortable %}
                             {{ column.name }}
                         {% else %}
@@ -22,7 +22,7 @@
         </thead>
         <tbody>
         {% for row in display_rows %}
-            <tr>
+            <tr{% if row.pk_path is not none %} data-row="{{ row.row_path }}"{% if row.row_label %} data-row-label="{{ row.row_label }}"{% endif %}{% endif %}>
                 {% for cell in row %}
                     <td class="col-{{ cell.column|to_css_class }} type-{{ cell.value_type }}">{{ cell.value }}</td>
                 {% endfor %}
@@ -31,6 +31,7 @@
         </tbody>
     </table>
 </div>
-{% else %}
+{% endif %}
+{% if not display_rows %}
     <p class="zero-results">0 records</p>
 {% endif %}

datasette/templates/allow_debug.html

@@ -3,29 +3,11 @@
 {% block title %}Debug allow rules{% endblock %}
 
 {% block extra_head %}
+{% include "_permission_ui_styles.html" %}
 <style>
-textarea {
-    height: 10em;
-    width: 95%;
-    box-sizing: border-box;
-    padding: 0.5em;
-    border: 2px dotted black;
-}
-.two-col {
-    display: inline-block;
-    width: 48%;
-}
-.two-col label {
-    width: 48%;
-}
 p.message-warning {
     white-space: pre-wrap;
 }
-@media only screen and (max-width: 576px) {
-    .two-col {
-        width: 100%;
-    }
-}
 </style>
 {% endblock %}
 
@@ -38,24 +20,28 @@ p.message-warning {
 
 <p>Use this tool to try out different actor and allow combinations. See <a href="https://docs.datasette.io/en/stable/authentication.html#defining-permissions-with-allow-blocks">Defining permissions with "allow" blocks</a> for documentation.</p>
 
-<form class="core" action="{{ urls.path('-/allow-debug') }}" method="get" style="margin-bottom: 1em">
+<div class="permission-form">
-    <div class="two-col">
+    <form class="core" action="{{ urls.path('-/allow-debug') }}" method="get">
-        <p><label>Allow block</label></p>
+        <div class="permission-form-grid">
-        <textarea name="allow">{{ allow_input }}</textarea>
+            <div class="form-section">
+                <label for="allow-block">Allow block</label>
+                <textarea class="permission-textarea" id="allow-block" name="allow">{{ allow_input }}</textarea>
             </div>
-    <div class="two-col">
+            <div class="form-section">
-        <p><label>Actor</label></p>
+                <label for="allow-actor">Actor</label>
-        <textarea name="actor">{{ actor_input }}</textarea>
+                <textarea class="permission-textarea" id="allow-actor" name="actor">{{ actor_input }}</textarea>
             </div>
-    <div style="margin-top: 1em;">
+        </div>
-        <input type="submit" value="Apply allow block to actor">
+        <div class="form-actions">
+            <button type="submit" class="submit-btn">Apply allow block to actor</button>
         </div>
     </form>
 
-{% if error %}<p class="message-warning">{{ error }}</p>{% endif %}
+    {% if error %}<p class="message-warning permission-form-result">{{ error }}</p>{% endif %}
 
-{% if result == "True" %}<p class="message-info">Result: allow</p>{% endif %}
+    {% if result == "True" %}<p class="message-info permission-form-result">Result: allow</p>{% endif %}
 
-{% if result == "False" %}<p class="message-error">Result: deny</p>{% endif %}
+    {% if result == "False" %}<p class="message-error permission-form-result">Result: deny</p>{% endif %}
+</div>
 
 {% endblock %}

datasette/templates/api_explorer.html

@@ -3,7 +3,6 @@
 {% block title %}API Explorer{% endblock %}
 
 {% block extra_head %}
-<script src="{{ base_url }}-/static/json-format-highlight-1.0.1.js"></script>
 {% endblock %}
 
 {% block content %}
@@ -19,7 +18,7 @@
 </p>
 <details open style="border: 2px solid #ccc; border-bottom: none; padding: 0.5em">
   <summary style="cursor: pointer;">GET</summary>
-  <form class="core" method="get" id="api-explorer-get" style="margin-top: 0.7em">
+  <form class="core" method="get" action="{{ urls.path('-/api') }}" id="api-explorer-get" style="margin-top: 0.7em">
     <div>
       <label for="path">API path:</label>
       <input type="text" id="path" name="path" style="width: 60%">
@@ -29,7 +28,7 @@
 </details>
 <details style="border: 2px solid #ccc; padding: 0.5em">
   <summary style="cursor: pointer">POST</summary>
-  <form class="core" method="post" id="api-explorer-post" style="margin-top: 0.7em">
+  <form class="core" method="post" action="{{ urls.path('-/api') }}" id="api-explorer-post" style="margin-top: 0.7em">
     <div>
       <label for="path">API path:</label>
       <input type="text" id="path" name="path" style="width: 60%">
@@ -126,7 +125,7 @@ getForm.addEventListener("submit", (ev) => {
     document.getElementById('response-status').textContent = response.status;
     return response.json();
   }).then((data) => {
-    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
+    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
     errorList.style.display = 'none';
   }).catch((error) => {
     alert(error);
@@ -174,7 +173,7 @@ postForm.addEventListener("submit", (ev) => {
     } else {
       errorList.style.display = 'none';
     }
-    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
+    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
     output.style.display = 'block';
   }).catch(err => {
     alert("Error: " + err);

datasette/templates/base.html

@@ -2,13 +2,14 @@
 <html lang="en">
 <head>
     <title>{% block title %}{% endblock %}</title>
-    <link rel="stylesheet" href="{{ urls.static('app.css') }}?{{ app_css_hash }}">
+    <link rel="stylesheet" href="{{ static('app.css') }}">
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
 {% for url in extra_css_urls %}
     <link rel="stylesheet" href="{{ url.url }}"{% if url.get("sri") %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}>
 {% endfor %}
 <script>window.datasetteVersion = '{{ datasette_version }}';</script>
-<script src="{{ urls.static('datasette-manager.js') }}" defer></script>
+<script src="{{ static('modal.js') }}" defer></script>
+<script src="{{ static('datasette-manager.js') }}" defer></script>
 {% for url in extra_js_urls %}
     <script {% if url.module %}type="module" {% endif %}src="{{ url.url }}"{% if url.get("sri") %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}></script>
 {% endfor %}
@@ -20,7 +21,7 @@
 <body class="{% block body_class %}{% endblock %}">
 <div class="not-footer">
 <header class="hd"><nav>{% block nav %}{% block crumbs %}{{ crumbs.nav(request=request) }}{% endblock %}
-    {% set links = menu_links() %}{% if links or show_logout %}
+    {% set links = menu_links() %}
     <details class="nav-menu details-menu">
         <summary><svg aria-labelledby="nav-menu-svg-title" role="img"
             fill="currentColor" stroke="currentColor" xmlns="http://www.w3.org/2000/svg"
@@ -29,20 +30,18 @@
                 <path fill-rule="evenodd" d="M1 2.75A.75.75 0 011.75 2h12.5a.75.75 0 110 1.5H1.75A.75.75 0 011 2.75zm0 5A.75.75 0 011.75 7h12.5a.75.75 0 110 1.5H1.75A.75.75 0 011 7.75zM1.75 12a.75.75 0 100 1.5h12.5a.75.75 0 100-1.5H1.75z"></path>
         </svg></summary>
         <div class="nav-menu-inner">
-            {% if links %}
             <ul>
+                <li><button type="button" class="button-as-link" data-navigation-search-open aria-haspopup="dialog" aria-expanded="false" aria-keyshortcuts="/">Jump to... <kbd class="keyboard-shortcut" aria-hidden="true" title="Keyboard shortcut: press / to open Jump to">/</kbd></button></li>
                 {% for link in links %}
                 <li><a href="{{ link.href }}">{{ link.label }}</a></li>
                 {% endfor %}
             </ul>
-            {% endif %}
             {% if show_logout %}
             <form class="nav-menu-logout" action="{{ urls.logout() }}" method="post">
-                <input type="hidden" name="csrftoken" value="{{ csrftoken() }}">
                 <button class="button-as-link">Log out</button>
             </form>{% endif %}
         </div>
-    </details>{% endif %}
+    </details>
     {% if actor %}
     <div class="actor">
         <strong>{{ display_actor(actor) }}</strong>
@@ -72,7 +71,7 @@
 {% endfor %}
 
 {% if select_templates %}<!-- Templates considered: {{ select_templates|join(", ") }} -->{% endif %}
-<script src="{{ urls.static('navigation-search.js') }}" defer></script>
+<script src="{{ static('navigation-search.js') }}" defer></script>
-<navigation-search url="/-/tables"></navigation-search>
+<navigation-search url="{{ urls.path("/-/jump") }}"></navigation-search>
 </body>
 </html>

datasette/templates/create_token.html

@@ -50,7 +50,6 @@
       </select>
     </div>
     <input type="text" name="expire_duration" style="width: 10%">
-    <input type="hidden" name="csrftoken" value="{{ csrftoken() }}">
     <input type="submit" value="Create token">
 
   <details style="margin-top: 1em" id="restrict-permissions">

datasette/templates/csrf_error.html

@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{% block title %}CSRF check failed){% endblock %}
+{% block title %}CSRF check failed{% endblock %}
 {% block content %}
 <h1>Form origin check failed</h1>
 
@@ -7,7 +7,7 @@
 
 <details><summary>Technical details</summary>
   <p>Developers: consult Datasette's <a href="https://docs.datasette.io/en/latest/internals.html#csrf-protection">CSRF protection documentation</a>.</p>
-  <p>Error code is {{ message_name }}.</p>
+  <p>Reason: {{ reason }}</p>
 </details>
 
 {% endblock %}