Write SQL query UI, canned queries now stored in internal database

PR #2741

.github/workflows/deploy-branch-preview.yml

@@ -1,35 +0,0 @@
-name: Deploy a Datasette branch preview to Vercel
-
-on:
-  workflow_dispatch:
-    inputs:
-      branch:
-        description: "Branch to deploy"
-        required: true
-        type: string
-
-jobs:
-  deploy-branch-preview:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v6
-      with:
-        python-version: "3.11"
-    - name: Install dependencies
-      run: |
-        pip install datasette-publish-vercel
-    - name: Deploy the preview
-      env:
-        VERCEL_TOKEN: ${{ secrets.BRANCH_PREVIEW_VERCEL_TOKEN }}
-      run: |
-        export BRANCH="${{ github.event.inputs.branch }}"
-        wget https://latest.datasette.io/fixtures.db
-        datasette publish vercel fixtures.db \
-          --branch $BRANCH \
-          --project "datasette-preview-$BRANCH" \
-          --token $VERCEL_TOKEN \
-          --scope datasette \
-          --about "Preview of $BRANCH" \
-          --about_url "https://github.com/simonw/datasette/tree/$BRANCH"

.github/workflows/deploy-latest.yml

@@ -2,10 +2,10 @@ name: Deploy latest.datasette.io
 
 on:
   workflow_dispatch:
-  # push:
-  #   branches:
-  #   - main
-  #   - 1.0-dev
+  push:
+    branches:
+    - main
+    #   - 1.0-dev
 
 permissions:
   contents: read
@@ -15,24 +15,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out datasette
-      uses: actions/checkout@v3
+      uses: actions/checkout@v6
     - name: Set up Python
       uses: actions/setup-python@v6
-      # Using Python 3.10 for gcloud compatibility:
       with:
-        python-version: "3.10"
-    - uses: actions/cache@v4
-      name: Configure pip caching
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/pyproject.toml') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        python-version: "3.13"
+        cache: pip
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -e .[test]
-        python -m pip install -e .[docs]
+        python -m pip install . --group dev
         python -m pip install sphinx-to-sqlite==0.1a1
     - name: Run tests
       if: ${{ github.ref == 'refs/heads/main' }}
@@ -65,7 +57,7 @@ jobs:
             db.route = "alternative-route"
         ' > plugins/alternative_route.py
         cp fixtures.db fixtures2.db
-    - name: And the counters writable canned query demo
+    - name: And the counters writable stored query demo
       run: |
         cat > plugins/counters.py <<EOF
         from datasette import hookimpl
@@ -77,23 +69,24 @@ jobs:
                 await db.execute_write("insert or ignore into counters (name, value) values ('counter_a', 0)")
                 await db.execute_write("insert or ignore into counters (name, value) values ('counter_b', 0)")
                 await db.execute_write("insert or ignore into counters (name, value) values ('counter_c', 0)")
-            return inner
-        @hookimpl
-        def canned_queries(database):
-            if database == "counters":
-                queries = {}
                 for name in ("counter_a", "counter_b", "counter_c"):
-                    queries["increment_{}".format(name)] = {
-                        "sql": "update counters set value = value + 1 where name = '{}'".format(name),
-                        "on_success_message_sql": "select 'Counter {name} incremented to ' || value from counters where name = '{name}'".format(name=name),
-                        "write": True,
-                    }
-                    queries["decrement_{}".format(name)] = {
-                        "sql": "update counters set value = value - 1 where name = '{}'".format(name),
-                        "on_success_message_sql": "select 'Counter {name} decremented to ' || value from counters where name = '{name}'".format(name=name),
-                        "write": True,
-                    }
-                return queries
+                    await datasette.add_query(
+                        "counters",
+                        "increment_{}".format(name),
+                        "update counters set value = value + 1 where name = '{}'".format(name),
+                        on_success_message_sql="select 'Counter {name} incremented to ' || value from counters where name = '{name}'".format(name=name),
+                        is_write=True,
+                        is_trusted=True,
+                    )
+                    await datasette.add_query(
+                        "counters",
+                        "decrement_{}".format(name),
+                        "update counters set value = value - 1 where name = '{}'".format(name),
+                        on_success_message_sql="select 'Counter {name} decremented to ' || value from counters where name = '{name}'".format(name=name),
+                        is_write=True,
+                        is_trusted=True,
+                    )
+            return inner
         EOF
     # - name: Make some modifications to metadata.json
     #   run: |
@@ -102,12 +95,13 @@ jobs:
     #       jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \
     #       > metadata.json
     #     cat metadata.json
-    - name: Set up Cloud Run
-      uses: google-github-actions/setup-gcloud@v0
+    - id: auth
+      name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v3
       with:
-        version: '318.0.0'
-        service_account_email: ${{ secrets.GCP_SA_EMAIL }}
-        service_account_key: ${{ secrets.GCP_SA_KEY }}
+        credentials_json: ${{ secrets.GCP_SA_KEY }}
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v3
     - name: Deploy to Cloud Run
       env:
         LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
@@ -123,7 +117,7 @@ jobs:
             --plugins-dir=plugins \
             --branch=$GITHUB_SHA \
             --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb" \
+            --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
             --install 'datasette-ephemeral-tables>=0.2.2' \
             --service "datasette-latest$SUFFIX" \
             --secret $LATEST_DATASETTE_SECRET

.github/workflows/documentation-links.yml

@@ -1,6 +1,6 @@
 name: Read the Docs Pull Request Preview
 on:
-  pull_request_target:
+  pull_request:
     types:
       - opened
 

.github/workflows/prettier.yml

@@ -10,8 +10,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out repo
-      uses: actions/checkout@v4
-    - uses: actions/cache@v4
+      uses: actions/checkout@v6
+    - uses: actions/cache@v5
       name: Configure npm caching
       with:
         path: ~/.npm

.github/workflows/publish.yml

@@ -14,7 +14,7 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
       with:
@@ -23,7 +23,7 @@ jobs:
         cache-dependency-path: pyproject.toml
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install . --group dev
     - name: Run tests
       run: |
         pytest
@@ -35,7 +35,7 @@ jobs:
     permissions:
       id-token: write
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -56,7 +56,7 @@ jobs:
     needs: [deploy]
     if: "!github.event.release.prerelease"
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -65,7 +65,7 @@ jobs:
         cache-dependency-path: pyproject.toml
     - name: Install dependencies
       run: |
-        python -m pip install -e .[docs]
+        python -m pip install . --group dev
         python -m pip install sphinx-to-sqlite==0.1a1
     - name: Build docs.db
       run: |-
@@ -73,12 +73,13 @@ jobs:
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
-    - name: Set up Cloud Run
-      uses: google-github-actions/setup-gcloud@v0
+    - id: auth
+      name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v2
       with:
-        version: '318.0.0'
-        service_account_email: ${{ secrets.GCP_SA_EMAIL }}
-        service_account_key: ${{ secrets.GCP_SA_KEY }}
+        credentials_json: ${{ secrets.GCP_SA_KEY }}
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v3
     - name: Deploy stable-docs.datasette.io to Cloud Run
       run: |-
         gcloud config set run/region us-central1
@@ -91,7 +92,7 @@ jobs:
     needs: [deploy]
     if: "!github.event.release.prerelease"
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Build and push to Docker Hub
       env:
         DOCKER_USER: ${{ secrets.DOCKER_USER }}

.github/workflows/push_docker_tag.yml

@@ -13,7 +13,7 @@ jobs:
   deploy_docker:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v6
     - name: Build and push to Docker Hub
       env:
         DOCKER_USER: ${{ secrets.DOCKER_USER }}

.github/workflows/spellcheck.yml

@@ -9,7 +9,7 @@ jobs:
   spellcheck:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -18,7 +18,7 @@ jobs:
         cache-dependency-path: '**/pyproject.toml'
     - name: Install dependencies
       run: |
-        pip install -e '.[docs]'
+        pip install . --group dev
     - name: Check spelling
       run: |
         codespell README.md --ignore-words docs/codespell-ignore-words.txt

.github/workflows/stable-docs.yml

@@ -0,0 +1,76 @@
+name: Update Stable Docs
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+    - main
+
+permissions:
+  contents: write
+
+jobs:
+  update_stable_docs:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v6
+      with:
+        fetch-depth: 0  # We need all commits to find docs/ changes
+    - name: Set up Git user
+      run: |
+        git config user.name "Automated"
+        git config user.email "actions@users.noreply.github.com"
+    - name: Create stable branch if it does not yet exist
+      run: |
+        if ! git ls-remote --heads origin stable | grep -qE '\bstable\b'; then
+          # Make sure we have all tags locally
+          git fetch --tags --quiet
+
+          # Latest tag that is just numbers and dots (optionally prefixed with 'v')
+          # e.g., 0.65.2 or v0.65.2 — excludes 1.0a20, 1.0-rc1, etc.
+          LATEST_RELEASE=$(
+            git tag -l --sort=-v:refname \
+            | grep -E '^v?[0-9]+(\.[0-9]+){1,3}$' \
+            | head -n1
+          )
+
+          git checkout -b stable
+
+          # If there are any stable releases, copy docs/ from the most recent
+          if [ -n "$LATEST_RELEASE" ]; then
+            rm -rf docs/
+            git checkout "$LATEST_RELEASE" -- docs/ || true
+          fi
+
+          git commit -m "Populate docs/ from $LATEST_RELEASE" || echo "No changes"
+          git push -u origin stable
+        fi
+    - name: Handle Release
+      if: github.event_name == 'release' && !github.event.release.prerelease
+      run: |
+        git fetch --all
+        git checkout stable
+        git reset --hard ${GITHUB_REF#refs/tags/}
+        git push origin stable --force
+    - name: Handle Commit to Main
+      if: contains(github.event.head_commit.message, '!stable-docs')
+      run: |
+        git fetch origin
+        git checkout -b stable origin/stable
+        # Get the list of modified files in docs/ from the current commit
+        FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/)
+        # Check if the list of files is non-empty
+        if [[ -n "$FILES" ]]; then
+          # Checkout those files to the stable branch to over-write with their contents
+          for FILE in $FILES; do
+            git checkout ${{ github.sha }} -- $FILE
+          done
+          git add docs/
+          git commit -m "Doc changes from ${{ github.sha }}"
+          git push origin stable
+        else
+          echo "No changes to docs/ in this commit."
+          exit 0
+        fi

.github/workflows/test-coverage.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out datasette
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -25,7 +25,7 @@ jobs:
     - name: Install Python dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install -e .[test]
+        python -m pip install . --group dev
         python -m pip install pytest-cov
     - name: Run tests
       run: |-

.github/workflows/test-pyodide.yml

@@ -12,7 +12,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Set up Python 3.10
       uses: actions/setup-python@v6
       with:
@@ -20,7 +20,7 @@ jobs:
         cache: 'pip'
         cache-dependency-path: '**/pyproject.toml'
     - name: Cache Playwright browsers
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       with:
         path: ~/.cache/ms-playwright/
         key: ${{ runner.os }}-browsers

.github/workflows/test-sqlite-support.yml

@@ -25,7 +25,7 @@ jobs:
           #"3.23.1" # 2018-04-10, before UPSERT
         ]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
       with:
@@ -45,7 +45,7 @@ jobs:
         (cd tests && gcc ext.c -fPIC -shared -o ext.so)
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install . --group dev
         pip freeze
     - name: Run tests
       run: |

.github/workflows/test.yml

@@ -12,7 +12,7 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
       with:
@@ -25,7 +25,7 @@ jobs:
         (cd tests && gcc ext.c -fPIC -shared -o ext.so)
     - name: Install dependencies
       run: |
-        pip install -e '.[test]'
+        pip install . --group dev
         pip freeze
     - name: Run tests
       run: |
@@ -33,11 +33,12 @@ jobs:
         pytest -m "serial"
         # And the test that exceeds a localhost HTTPS server
         tests/test_datasette_https_server.sh
-    - name: Install docs dependencies
-      run: |
-        pip install -e '.[docs]'
     - name: Black
-      run: black --check .
+      run: |
+        black --version
+        black --check .
+    - name: Ruff
+      run: ruff check datasette tests
     - name: Check if cog needs to be run
       run: |
         cog --check docs/*.rst

.github/workflows/tmate-mac.yml

@@ -10,6 +10,6 @@ jobs:
   build:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v6
     - name: Setup tmate session
       uses: mxschmitt/action-tmate@v3

.github/workflows/tmate.yml

@@ -11,7 +11,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v6
     - name: Setup tmate session
       uses: mxschmitt/action-tmate@v3
       env:

.gitignore

@@ -8,6 +8,9 @@ scratchpad
 uv.lock
 data.db
 
+# test databases
+*.db
+
 # We don't use Pipfile, so ignore them
 Pipfile
 Pipfile.lock
@@ -127,3 +130,5 @@ node_modules
 tests/*.dylib
 tests/*.so
 tests/*.dll
+
+.idea

.readthedocs.yaml

@@ -1,16 +1,17 @@
 version: 2
 
-build:
-  os: ubuntu-20.04
-  tools:
-    python: "3.11"
-
 sphinx:
-   configuration: docs/conf.py
+  configuration: docs/conf.py
 
-python:
-  install:
-  - method: pip
-    path: .
-    extra_requirements:
-    - docs
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+  jobs:
+    install:
+    - pip install --upgrade pip
+    - pip install . --group dev
+
+formats:
+- pdf
+- epub

Justfile

@@ -5,7 +5,7 @@ export DATASETTE_SECRET := "not_a_secret"
 
 # Setup project
 @init:
-  uv sync --extra test --extra docs
+  uv sync
 
 # Run pytest with supplied options
 @test *options: init
@@ -17,19 +17,23 @@ export DATASETTE_SECRET := "not_a_secret"
   uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt
   uv run codespell tests --ignore-words docs/codespell-ignore-words.txt
 
-# Run linters: black, flake8, mypy, cog
+# Run linters: black, ruff, cog
 @lint: codespell
-  uv run black . --check
-  uv run flake8
+  uv run black datasette tests --check
+  uv run ruff check datasette tests
   uv run cog --check README.md docs/*.rst
 
+# Apply ruff fixes
+@fix:
+  uv run ruff check --fix datasette tests
+
 # Rebuild docs with cog
 @cog:
   uv run cog -r README.md docs/*.rst
 
 # Serve live docs on localhost:8000
 @docs: cog blacken-docs
-  uv sync --extra docs && cd docs && uv run make livehtml
+  uv run make -C docs livehtml
 
 # Build docs as static HTML
 @docs-build: cog blacken-docs
@@ -37,7 +41,7 @@ export DATASETTE_SECRET := "not_a_secret"
 
 # Apply Black
 @black:
-  uv run black .
+  uv run black datasette tests
 
 # Apply blacken-docs
 @blacken-docs:

datasette/__init__.py

@@ -1,6 +1,7 @@
 from datasette.permissions import Permission  # noqa
 from datasette.version import __version_info__, __version__  # noqa
 from datasette.events import Event  # noqa
+from datasette.tokens import TokenHandler, TokenRestrictions  # noqa
 from datasette.utils.asgi import Forbidden, NotFound, Request, Response  # noqa
 from datasette.utils import actor_matches_allow  # noqa
 from datasette.views import Context  # noqa

datasette/_pytest_plugin.py

@@ -0,0 +1,108 @@
+"""
+Pytest plugin that automatically closes any Datasette instances constructed
+during a pytest test — both in the test body and in function-scoped
+fixtures. Instances constructed by session-, module-, class- or package-
+scoped fixtures are left alone, because other tests in the session will
+still want to use them.
+
+Registered as a pytest11 entry point in pyproject.toml so that downstream
+projects using Datasette get the same FD-safety net for their own tests.
+
+Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the
+equivalent ini file).
+"""
+
+from __future__ import annotations
+
+import contextvars
+import weakref
+
+import pytest
+
+from datasette.app import Datasette
+
+_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar(
+    "datasette_active_instances", default=None
+)
+
+_original_init = Datasette.__init__
+
+
+def _tracking_init(self, *args, **kwargs):
+    _original_init(self, *args, **kwargs)
+    instances = _active_instances.get()
+    if instances is not None:
+        instances.append(weakref.ref(self))
+
+
+Datasette.__init__ = _tracking_init
+
+
+def pytest_addoption(parser):
+    parser.addini(
+        "datasette_autoclose",
+        help=(
+            "Automatically close Datasette instances created inside test "
+            "bodies and function-scoped fixtures (default: true)."
+        ),
+        default="true",
+    )
+
+
+def _enabled(config) -> bool:
+    value = config.getini("datasette_autoclose")
+    if isinstance(value, bool):
+        return value
+    return str(value).strip().lower() not in ("false", "0", "no", "off")
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_protocol(item, nextitem):
+    """Track Datasette instances across setup, call and teardown; close at end."""
+    if not _enabled(item.config):
+        yield
+        return
+    refs: list[weakref.ref] = []
+    token = _active_instances.set(refs)
+    try:
+        yield
+    finally:
+        _active_instances.reset(token)
+        for ref in reversed(refs):
+            ds = ref()
+            if ds is None:
+                continue
+            try:
+                ds.close()
+            except Exception as e:
+                item.warn(
+                    pytest.PytestUnraisableExceptionWarning(
+                        f"Error closing Datasette instance: {e!r}"
+                    )
+                )
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_fixture_setup(fixturedef, request):
+    """Exempt instances created by non-function-scoped fixtures.
+
+    Session-, module-, class- and package-scoped fixtures produce Datasette
+    instances that must survive beyond the current test — other tests in
+    the session will still use them. When such a fixture creates one or
+    more Datasette instances during its setup, we snapshot the tracking
+    list before the fixture runs and subtract off any instances that were
+    added during its setup, so they don't get closed at test teardown.
+    """
+    refs = _active_instances.get()
+    if refs is None:
+        yield
+        return
+    before_ids = {id(ref) for ref in refs}
+    yield
+    if fixturedef.scope != "function":
+        new_refs = [ref for ref in refs if id(ref) not in before_ids]
+        for new_ref in new_refs:
+            try:
+                refs.remove(new_ref)
+            except ValueError:
+                pass

datasette/cli.py

@@ -109,15 +109,11 @@ def sqlite_extensions(fn):
             return fn(*args, **kwargs)
         except AttributeError as e:
             if "enable_load_extension" in str(e):
-                raise click.ClickException(
-                    textwrap.dedent(
-                        """
+                raise click.ClickException(textwrap.dedent("""
                     Your Python installation does not have the ability to load SQLite extensions.
 
                     More information: https://datasette.io/help/extensions
-                    """
-                    ).strip()
-                )
+                    """).strip())
             raise
 
     return wrapped
@@ -438,10 +434,20 @@ def uninstall(packages, yes):
     help="Output URL that sets a cookie authenticating the root user",
     is_flag=True,
 )
+@click.option(
+    "--default-deny",
+    help="Deny all permissions by default",
+    is_flag=True,
+)
 @click.option(
     "--get",
     help="Run an HTTP GET request against this path, print results and exit",
 )
+@click.option(
+    "--headers",
+    is_flag=True,
+    help="Include HTTP headers in --get output",
+)
 @click.option(
     "--token",
     help="API token to send with --get requests",
@@ -509,7 +515,9 @@ def serve(
     settings,
     secret,
     root,
+    default_deny,
     get,
+    headers,
     token,
     actor,
     version_note,
@@ -539,7 +547,7 @@ def serve(
     if reload:
         import hupper
 
-        reloader = hupper.start_reloader("datasette.cli.serve")
+        reloader = hupper.start_reloader("datasette.cli.cli")
         if immutable:
             reloader.watch_files(immutable)
         if config:
@@ -588,6 +596,7 @@ def serve(
         crossdb=crossdb,
         nolock=nolock,
         internal=internal,
+        default_deny=default_deny,
     )
 
     # Separate directories from files
@@ -606,7 +615,9 @@ def serve(
     for file in file_paths:
         if not pathlib.Path(file).exists():
             if create:
-                sqlite3.connect(file).execute("vacuum")
+                conn = sqlite3.connect(file)
+                conn.execute("vacuum")
+                conn.close()
             else:
                 raise click.ClickException(
                     "Invalid value for '[FILES]...': Path '{}' does not exist.".format(
@@ -652,25 +663,43 @@ def serve(
         # Private utility mechanism for writing unit tests
         return ds
 
-    # Run the "startup" plugin hooks
-    run_sync(ds.invoke_startup)
-
-    # Run async soundness checks - but only if we're not under pytest
+    # Run async soundness checks before startup hooks, since invoke_startup
+    # now populates internal tables which requires querying each database
     run_sync(lambda: check_databases(ds))
 
+    # Run the "startup" plugin hooks
+    try:
+        run_sync(ds.invoke_startup)
+    except StartupError as e:
+        raise click.ClickException(e.args[0])
+
+    if headers and not get:
+        raise click.ClickException("--headers can only be used with --get")
+
     if token and not get:
         raise click.ClickException("--token can only be used with --get")
 
     if get:
         client = TestClient(ds)
-        headers = {}
+        request_headers = {}
         if token:
-            headers["Authorization"] = "Bearer {}".format(token)
+            request_headers["Authorization"] = "Bearer {}".format(token)
         cookies = {}
         if actor:
             cookies["ds_actor"] = client.actor_cookie(json.loads(actor))
-        response = client.get(get, headers=headers, cookies=cookies)
-        click.echo(response.text)
+        response = client.get(get, headers=request_headers, cookies=cookies)
+
+        if headers:
+            # Output HTTP status code, headers, two newlines, then the response body
+            click.echo(f"HTTP/1.1 {response.status}")
+            for key, value in response.headers.items():
+                click.echo(f"{key}: {value}")
+            if response.text:
+                click.echo()
+                click.echo(response.text)
+        else:
+            click.echo(response.text)
+
         exit_code = 0 if response.status == 200 else 1
         sys.exit(exit_code)
         return
@@ -788,7 +817,10 @@ def create_token(
     ds = Datasette(secret=secret, plugins_dir=plugins_dir)
 
     # Run ds.invoke_startup() in an event loop
-    run_sync(ds.invoke_startup)
+    try:
+        run_sync(ds.invoke_startup)
+    except StartupError as e:
+        raise click.ClickException(e.args[0])
 
     # Warn about any unknown actions
     actions = []
@@ -803,21 +835,23 @@ def create_token(
                 err=True,
             )
 
-    restrict_database = {}
-    for database, action in databases:
-        restrict_database.setdefault(database, []).append(action)
-    restrict_resource = {}
-    for database, resource, action in resources:
-        restrict_resource.setdefault(database, {}).setdefault(resource, []).append(
-            action
-        )
+    from datasette.tokens import TokenRestrictions
 
-    token = ds.create_token(
-        id,
-        expires_after=expires_after,
-        restrict_all=alls,
-        restrict_database=restrict_database,
-        restrict_resource=restrict_resource,
+    restrictions = TokenRestrictions()
+    for action in alls:
+        restrictions.allow_all(action)
+    for database, action in databases:
+        restrictions.allow_database(database, action)
+    for database, resource, action in resources:
+        restrictions.allow_resource(database, resource, action)
+
+    token = run_sync(
+        lambda: ds.create_token(
+            id,
+            expires_after=expires_after,
+            restrictions=restrictions,
+            handler="signed",
+        )
     )
     click.echo(token)
     if debug:

datasette/column_types.py

@@ -0,0 +1,83 @@
+from enum import Enum
+
+
+class SQLiteType(Enum):
+    TEXT = "TEXT"
+    INTEGER = "INTEGER"
+    REAL = "REAL"
+    BLOB = "BLOB"
+    NULL = "NULL"
+
+    @classmethod
+    def from_declared_type(cls, declared_type: str | None) -> "SQLiteType | None":
+        if declared_type is None:
+            return cls.NULL
+
+        normalized = declared_type.strip().upper()
+        if not normalized:
+            return cls.NULL
+
+        if normalized == cls.NULL.value:
+            return cls.NULL
+        if "INT" in normalized:
+            return cls.INTEGER
+        if any(token in normalized for token in ("CHAR", "CLOB", "TEXT")):
+            return cls.TEXT
+        if "BLOB" in normalized:
+            return cls.BLOB
+        if any(
+            token in normalized
+            for token in ("REAL", "FLOA", "DOUB")  # codespell:ignore doub
+        ):
+            return cls.REAL
+
+        return None
+
+
+class ColumnType:
+    """
+    Base class for column types.
+
+    Subclasses must define ``name`` and ``description`` as class attributes:
+
+    - ``name``: Unique identifier string. Lowercase, no spaces.
+      Examples: "markdown", "file", "email", "url", "point", "image".
+    - ``description``: Human-readable label for admin UI dropdowns.
+      Examples: "Markdown text", "File reference", "Email address".
+    - ``sqlite_types``: Optional tuple of SQLiteType values restricting
+      which SQLite column types this ColumnType can be assigned to.
+
+    Instantiate with an optional ``config`` dict to bind per-column
+    configuration::
+
+        ct = MyColumnType(config={"key": "value"})
+        ct.config  # {"key": "value"}
+    """
+
+    name: str
+    description: str
+    sqlite_types: tuple[SQLiteType, ...] | None = None
+
+    def __init__(self, config=None):
+        self.config = config
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        """
+        Return an HTML string to render this cell value, or None to
+        fall through to the default render_cell plugin hook chain.
+        """
+        return None
+
+    async def validate(self, value, datasette):
+        """
+        Validate a value before it is written. Return None if valid,
+        or a string error message if invalid.
+        """
+        return None
+
+    async def transform_value(self, value, datasette):
+        """
+        Transform a value before it appears in JSON API output.
+        Return the transformed value. Default: return unchanged.
+        """
+        return value

datasette/csrf.py

@@ -0,0 +1,178 @@
+"""
+Header-based CSRF (Cross-Origin) protection.
+
+Datasette uses the Sec-Fetch-Site + Origin header approach described in
+Filippo Valsorda's article (https://words.filippo.io/csrf/) and implemented
+in Go 1.25's http.CrossOriginProtection. This replaces the previous
+token-based asgi-csrf mechanism.
+"""
+
+from __future__ import annotations
+
+import secrets
+import urllib.parse
+
+from .utils.asgi import asgi_send
+
+SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
+
+DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443}
+
+
+def _normalize_headers(raw_headers):
+    """Lowercase header names; for duplicates, last value wins."""
+    result = {}
+    for name, value in raw_headers:
+        if isinstance(name, str):
+            name = name.encode("latin-1")
+        if isinstance(value, str):
+            value = value.encode("latin-1")
+        result[name.lower()] = value
+    return result
+
+
+def _origin_tuple(value):
+    """
+    Parse an origin-like string into ``(scheme, host, port)`` with default
+    ports filled in. Raises ``ValueError`` for malformed input.
+    """
+    parsed = urllib.parse.urlsplit(value)
+    scheme = (parsed.scheme or "").lower()
+    host = (parsed.hostname or "").lower()
+    if not scheme or not host:
+        raise ValueError("missing scheme or host in {!r}".format(value))
+    port = parsed.port  # may raise ValueError on bad ports
+    if port is None:
+        port = DEFAULT_PORTS.get(scheme)
+    if port is None:
+        raise ValueError("unknown default port for scheme {!r}".format(scheme))
+    return scheme, host, port
+
+
+def _install_legacy_csrftoken(scope):
+    """
+    Populate ``scope["csrftoken"]`` with a callable returning a per-request
+    random token. Provided for plugin compatibility only - core no longer
+    uses this value for CSRF enforcement.
+    """
+
+    def csrftoken():
+        if "_datasette_legacy_csrftoken" not in scope:
+            scope["_datasette_legacy_csrftoken"] = secrets.token_urlsafe(32)
+        return scope["_datasette_legacy_csrftoken"]
+
+    scope["csrftoken"] = csrftoken
+
+
+class CrossOriginProtectionMiddleware:
+    """
+    Modern CSRF protection using the Sec-Fetch-Site and Origin headers.
+
+    Based on Filippo Valsorda's algorithm, as implemented in Go 1.25's
+    http.CrossOriginProtection. See https://words.filippo.io/csrf/
+
+    Unsafe-method requests are allowed through only if they look same-origin.
+    Non-browser clients (curl, etc.) send neither Sec-Fetch-Site nor Origin
+    and are passed through unchanged - CSRF is a browser-only attack.
+    """
+
+    SAFE_METHODS = SAFE_METHODS
+
+    def __init__(self, app, datasette):
+        self.app = app
+        self.datasette = datasette
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        _install_legacy_csrftoken(scope)
+
+        if scope.get("method", "GET") in self.SAFE_METHODS:
+            await self.app(scope, receive, send)
+            return
+
+        headers = _normalize_headers(scope.get("headers") or [])
+
+        authorization = headers.get(b"authorization", b"").decode("latin-1")
+        cookie_header = headers.get(b"cookie")
+        # Bearer-token requests are not ambient browser credentials, so they
+        # are not CSRF-vulnerable. Narrowly exempt them from the header check
+        # before evaluating Sec-Fetch-Site / Origin. Only "Bearer" is exempt;
+        # schemes like Basic or Digest can be browser-managed and ambient.
+        # If the request also carries a Cookie header, ambient cookie auth
+        # could be in play, so do NOT treat it as exempt.
+        if authorization and not cookie_header:
+            parts = authorization.split(None, 1)
+            if parts and parts[0].lower() == "bearer":
+                await self.app(scope, receive, send)
+                return
+
+        origin_bytes = headers.get(b"origin")
+        sec_fetch_site_bytes = headers.get(b"sec-fetch-site")
+        host_bytes = headers.get(b"host", b"")
+        origin = origin_bytes.decode("latin-1") if origin_bytes else None
+        sec_fetch_site = (
+            sec_fetch_site_bytes.decode("latin-1") if sec_fetch_site_bytes else None
+        )
+        host = host_bytes.decode("latin-1")
+
+        # Primary defense: Sec-Fetch-Site (set by browsers, unforgeable from JS)
+        if sec_fetch_site is not None:
+            if sec_fetch_site in ("same-origin", "none"):
+                await self.app(scope, receive, send)
+                return
+            await self._forbid(
+                send,
+                "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format(
+                    sec_fetch_site
+                ),
+            )
+            return
+
+        # No Sec-Fetch-Site and no Origin -> non-browser client (curl, API, etc.)
+        if origin is None:
+            await self.app(scope, receive, send)
+            return
+
+        # Fallback for older browsers: Origin must match the request's own
+        # scheme + host + port. Compare full origin tuples, not host alone.
+        request_scheme = self._request_scheme(scope)
+        try:
+            origin_tuple = _origin_tuple(origin)
+            expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host))
+        except ValueError:
+            await self._forbid(
+                send,
+                "Malformed Origin {!r} or Host {!r}".format(origin, host),
+            )
+            return
+
+        if origin_tuple == expected_tuple:
+            await self.app(scope, receive, send)
+            return
+
+        await self._forbid(
+            send,
+            "Origin {!r} does not match Host {!r}".format(origin, host),
+        )
+
+    def _request_scheme(self, scope):
+        if self.datasette is not None:
+            try:
+                if self.datasette.setting("force_https_urls"):
+                    return "https"
+            except Exception:
+                pass
+        return scope.get("scheme") or "http"
+
+    async def _forbid(self, send, reason):
+        await asgi_send(
+            send,
+            content=await self.datasette.render_template(
+                "csrf_error.html", {"reason": reason}
+            ),
+            status=403,
+            content_type="text/html; charset=utf-8",
+        )

datasette/database.py

@@ -1,15 +1,19 @@
 import asyncio
+import atexit
 from collections import namedtuple
+import inspect
+import os
 from pathlib import Path
-import janus
 import queue
 import sqlite_utils
 import sys
+import tempfile
 import threading
 import uuid
 
 from .tracer import trace
 from .utils import (
+    call_with_supported_arguments,
     detect_fts,
     detect_primary_keys,
     detect_spatialite,
@@ -21,6 +25,7 @@ from .utils import (
     table_columns,
     table_column_details,
 )
+from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables
 from .utils.sqlite import sqlite_version
 from .inspect import inspect_hash
 
@@ -29,6 +34,13 @@ connections = threading.local()
 AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file"))
 
 
+class DatasetteClosedError(RuntimeError):
+    """Raised when using a Datasette or Database instance after close()."""
+
+
+_SHUTDOWN = object()
+
+
 class Database:
     # For table counts stop at this many rows:
     count_limit = 10000
@@ -42,6 +54,7 @@ class Database:
         is_memory=False,
         memory_name=None,
         mode=None,
+        is_temp_disk=False,
     ):
         self.name = None
         self._thread_local_id = f"x{self._thread_local_id_counter}"
@@ -52,19 +65,44 @@ class Database:
         self.is_mutable = is_mutable
         self.is_memory = is_memory
         self.memory_name = memory_name
+        self.is_temp_disk = is_temp_disk
         if memory_name is not None:
             self.is_memory = True
+        if is_temp_disk:
+            fd, temp_path = tempfile.mkstemp(suffix=".db", prefix="datasette_temp_")
+            os.close(fd)
+            self.path = temp_path
+            self.is_mutable = True
+            self.mode = "rwc"
+            self._wal_enabled = False
+            atexit.register(self._cleanup_temp_file)
+        else:
+            self._wal_enabled = False
         self.cached_hash = None
         self.cached_size = None
         self._cached_table_counts = None
         self._write_thread = None
         self._write_queue = None
+        self._closed = False
+        self._pending_execute_futures = set()
+        self._pending_execute_futures_lock = threading.Lock()
         # These are used when in non-threaded mode:
         self._read_connection = None
         self._write_connection = None
         # This is used to track all file connections so they can be closed
         self._all_file_connections = []
-        self.mode = mode
+        if not is_temp_disk:
+            self.mode = mode
+
+    def _check_not_closed(self):
+        if self._closed:
+            raise DatasetteClosedError(
+                "Database {!r} has been closed".format(self.name)
+            )
+
+    def _remove_pending_execute_future(self, future):
+        with self._pending_execute_futures_lock:
+            self._pending_execute_futures.discard(future)
 
     @property
     def cached_table_counts(self):
@@ -85,6 +123,8 @@ class Database:
         return md5_not_usedforsecurity(self.name)[:6]
 
     def suggest_name(self):
+        if self.is_temp_disk:
+            return "_temp_disk"
         if self.path:
             return Path(self.path).stem
         elif self.memory_name:
@@ -123,32 +163,104 @@ class Database:
             f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs
         )
         self._all_file_connections.append(conn)
+        if self.is_temp_disk and not self._wal_enabled:
+            conn.execute("PRAGMA journal_mode=WAL")
+            self._wal_enabled = True
         return conn
 
     def close(self):
-        # Close all connections - useful to avoid running out of file handles in tests
-        for connection in self._all_file_connections:
-            connection.close()
+        """Release all resources held by this database.
+
+        Idempotent. After close() further calls to execute()/execute_fn()/
+        execute_write()/execute_write_fn() raise DatasetteClosedError.
+        """
+        if self._closed:
+            return
+        with self._pending_execute_futures_lock:
+            if self._closed:
+                return
+            self._closed = True
+            pending_execute_futures = tuple(self._pending_execute_futures)
+        # Shut down the write thread, if any, via a sentinel. The thread
+        # drains any writes already queued before the sentinel and then
+        # closes its own write connection and returns.
+        write_thread = self._write_thread
+        if write_thread is not None and self._write_queue is not None:
+            self._write_queue.put(_SHUTDOWN)
+            write_thread.join(timeout=10)
+            if write_thread.is_alive():
+                sys.stderr.write(
+                    "Datasette: write thread for {!r} did not exit within 10s\n".format(
+                        self.name
+                    )
+                )
+                sys.stderr.flush()
+        for future in pending_execute_futures:
+            try:
+                future.result()
+            except Exception:
+                pass
+        # Close anything still tracked in _all_file_connections
+        for connection in self._all_file_connections:
+            try:
+                connection.close()
+            except Exception:
+                pass
+        self._all_file_connections = []
+        # Drop per-thread cached read connections we can reach
+        try:
+            delattr(connections, self._thread_local_id)
+        except AttributeError:
+            pass
+        # Close non-threaded-mode cached connections if still open
+        if self._read_connection is not None:
+            try:
+                self._read_connection.close()
+            except Exception:
+                pass
+            self._read_connection = None
+        if self._write_connection is not None:
+            try:
+                self._write_connection.close()
+            except Exception:
+                pass
+            self._write_connection = None
+        if self.is_temp_disk:
+            self._cleanup_temp_file()
+
+    def _cleanup_temp_file(self):
+        if self.is_temp_disk and self.path:
+            for suffix in ("", "-wal", "-shm"):
+                try:
+                    os.unlink(self.path + suffix)
+                except OSError:
+                    pass
+
+    async def execute_write(self, sql, params=None, block=True, request=None):
+        self._check_not_closed()
 
-    async def execute_write(self, sql, params=None, block=True):
         def _inner(conn):
             return conn.execute(sql, params or [])
 
         with trace("sql", database=self.name, sql=sql.strip(), params=params):
-            results = await self.execute_write_fn(_inner, block=block)
+            results = await self.execute_write_fn(_inner, block=block, request=request)
         return results
 
-    async def execute_write_script(self, sql, block=True):
+    async def execute_write_script(self, sql, block=True, request=None):
+        self._check_not_closed()
+
         def _inner(conn):
             return conn.executescript(sql)
 
         with trace("sql", database=self.name, sql=sql.strip(), executescript=True):
             results = await self.execute_write_fn(
-                _inner, block=block, transaction=False
+                _inner, block=block, transaction=False, request=request
             )
         return results
 
-    async def execute_write_many(self, sql, params_seq, block=True):
+    async def execute_write_many(self, sql, params_seq, block=True, request=None):
+        self._check_not_closed()
+
         def _inner(conn):
             count = 0
 
@@ -163,11 +275,14 @@ class Database:
         with trace(
             "sql", database=self.name, sql=sql.strip(), executemany=True
         ) as kwargs:
-            results, count = await self.execute_write_fn(_inner, block=block)
+            results, count = await self.execute_write_fn(
+                _inner, block=block, request=request
+            )
             kwargs["count"] = count
         return results
 
     async def execute_isolated_fn(self, fn):
+        self._check_not_closed()
         # Open a new connection just for the duration of this function
         # blocking the write queue to avoid any writes occurring during it
         if self.ds.executor is None:
@@ -187,7 +302,21 @@ class Database:
             # Threaded mode - send to write thread
             return await self._send_to_write_thread(fn, isolated_connection=True)
 
-    async def execute_write_fn(self, fn, block=True, transaction=True):
+    async def analyze_sql(self, sql, params=None) -> SQLAnalysis:
+        self._check_not_closed()
+
+        return await self.execute_isolated_fn(
+            lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name)
+        )
+
+    async def execute_write_fn(self, fn, block=True, transaction=True, request=None):
+        self._check_not_closed()
+        pending_events = []
+
+        def track_event(event):
+            pending_events.append(event)
+
+        fn = self._wrap_fn_with_hooks(fn, request, transaction, track_event)
         if self.ds.executor is None:
             # non-threaded mode
             if self._write_connection is None:
@@ -195,13 +324,67 @@ class Database:
                 self.ds._prepare_connection(self._write_connection, self.name)
             if transaction:
                 with self._write_connection:
-                    return fn(self._write_connection)
+                    result = fn(self._write_connection)
             else:
-                return fn(self._write_connection)
+                result = fn(self._write_connection)
         else:
-            return await self._send_to_write_thread(
+            result = await self._send_to_write_thread(
                 fn, block=block, transaction=transaction
             )
+        if block:
+            for event in pending_events:
+                await self.ds.track_event(event)
+        else:
+            # For non-blocking writes, spawn a background task to
+            # dispatch events after the write thread completes
+            task_id, reply_future = result
+
+            async def _dispatch_events_after_write():
+                try:
+                    await reply_future
+                except Exception:
+                    # if the write failed, don't emit success events
+                    return
+                for event in pending_events:
+                    await self.ds.track_event(event)
+
+            asyncio.ensure_future(_dispatch_events_after_write())
+            result = task_id
+        return result
+
+    def _wrap_fn_with_hooks(self, fn, request, transaction, track_event):
+        from .plugins import pm
+
+        # Wrap fn so it receives track_event if its signature supports it.
+        # Historically fn was called positionally, so any single-parameter
+        # name (conn, connection, db, ...) worked. Preserve that by only
+        # switching to keyword dependency injection when the callback
+        # explicitly opts in by declaring a `track_event` parameter.
+        original_fn = fn
+
+        if "track_event" in inspect.signature(original_fn).parameters:
+
+            def fn_with_track_event(conn):
+                return call_with_supported_arguments(
+                    original_fn, conn=conn, track_event=track_event
+                )
+
+            fn = fn_with_track_event
+
+        wrappers = pm.hook.write_wrapper(
+            datasette=self.ds,
+            database=self.name,
+            request=request,
+            transaction=transaction,
+        )
+        wrappers = [w for w in wrappers if w is not None]
+        if not wrappers:
+            return fn
+        # Build the wrapped fn by nesting context manager generators.
+        # The first wrapper returned by pluggy is outermost.
+        for wrapper_factory in reversed(wrappers):
+            fn = _apply_write_wrapper(fn, wrapper_factory, track_event)
+        return fn
 
     async def _send_to_write_thread(
         self, fn, block=True, isolated_connection=False, transaction=True
@@ -217,18 +400,15 @@ class Database:
             )
             self._write_thread.start()
         task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
-        reply_queue = janus.Queue()
+        loop = asyncio.get_running_loop()
+        reply_future = loop.create_future()
         self._write_queue.put(
-            WriteTask(fn, task_id, reply_queue, isolated_connection, transaction)
+            WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction)
         )
         if block:
-            result = await reply_queue.async_q.get()
-            if isinstance(result, Exception):
-                raise result
-            else:
-                return result
+            return await reply_future
         else:
-            return task_id
+            return task_id, reply_future
 
     def _execute_writes(self):
         # Infinite looping thread that protects the single write connection
@@ -242,38 +422,47 @@ class Database:
             conn_exception = e
         while True:
             task = self._write_queue.get()
+            if task is _SHUTDOWN:
+                if conn is not None:
+                    try:
+                        conn.close()
+                    except Exception:
+                        pass
+                return
+            exception = None
+            result = None
             if conn_exception is not None:
-                result = conn_exception
+                exception = conn_exception
+            elif task.isolated_connection:
+                isolated_connection = self.connect(write=True)
+                try:
+                    result = task.fn(isolated_connection)
+                except Exception as e:
+                    sys.stderr.write("{}\n".format(e))
+                    sys.stderr.flush()
+                    exception = e
+                finally:
+                    isolated_connection.close()
+                    try:
+                        self._all_file_connections.remove(isolated_connection)
+                    except ValueError:
+                        # Was probably a memory connection
+                        pass
             else:
-                if task.isolated_connection:
-                    isolated_connection = self.connect(write=True)
-                    try:
-                        result = task.fn(isolated_connection)
-                    except Exception as e:
-                        sys.stderr.write("{}\n".format(e))
-                        sys.stderr.flush()
-                        result = e
-                    finally:
-                        isolated_connection.close()
-                        try:
-                            self._all_file_connections.remove(isolated_connection)
-                        except ValueError:
-                            # Was probably a memory connection
-                            pass
-                else:
-                    try:
-                        if task.transaction:
-                            with conn:
-                                result = task.fn(conn)
-                        else:
+                try:
+                    if task.transaction:
+                        with conn:
                             result = task.fn(conn)
-                    except Exception as e:
-                        sys.stderr.write("{}\n".format(e))
-                        sys.stderr.flush()
-                        result = e
-            task.reply_queue.sync_q.put(result)
+                    else:
+                        result = task.fn(conn)
+                except Exception as e:
+                    sys.stderr.write("{}\n".format(e))
+                    sys.stderr.flush()
+                    exception = e
+            _deliver_write_result(task, result, exception)
 
     async def execute_fn(self, fn):
+        self._check_not_closed()
         if self.ds.executor is None:
             # non-threaded mode
             if self._read_connection is None:
@@ -290,9 +479,12 @@ class Database:
                 setattr(connections, self._thread_local_id, conn)
             return fn(conn)
 
-        return await asyncio.get_event_loop().run_in_executor(
-            self.ds.executor, in_thread
-        )
+        with self._pending_execute_futures_lock:
+            self._check_not_closed()
+            future = self.ds.executor.submit(in_thread)
+            self._pending_execute_futures.add(future)
+        future.add_done_callback(self._remove_pending_execute_future)
+        return await asyncio.wrap_future(future)
 
     async def execute(
         self,
@@ -304,6 +496,7 @@ class Database:
         log_sql_errors=True,
     ):
         """Executes sql against db_name in a thread"""
+        self._check_not_closed()
         page_size = page_size or self.ds.page_size
 
         def sql_operation_in_thread(conn):
@@ -351,7 +544,7 @@ class Database:
     def hash(self):
         if self.cached_hash is not None:
             return self.cached_hash
-        elif self.is_mutable or self.is_memory:
+        elif self.is_mutable or self.is_memory or self.is_temp_disk:
             return None
         elif self.ds.inspect_data and self.ds.inspect_data.get(self.name):
             self.cached_hash = self.ds.inspect_data[self.name]["hash"]
@@ -431,7 +624,7 @@ class Database:
 
     async def table_names(self):
         results = await self.execute(
-            "select name from sqlite_master where type='table'"
+            "select name from sqlite_master where type='table' order by name"
         )
         return [r[0] for r in results.rows]
 
@@ -510,10 +703,7 @@ class Database:
             ]
 
         if sqlite_version()[1] >= 37:
-            hidden_tables += [
-                x[0]
-                for x in await self.execute(
-                    """
+            hidden_tables += [x[0] for x in await self.execute("""
                       with shadow_tables as (
                         select name
                         from pragma_table_list
@@ -532,14 +722,9 @@ class Database:
                         select name from core_tables
                       )
                       select name from combined order by 1
-                    """
-                )
-            ]
+                    """)]
         else:
-            hidden_tables += [
-                x[0]
-                for x in await self.execute(
-                    """
+            hidden_tables += [x[0] for x in await self.execute("""
                       WITH base AS (
                         SELECT name
                         FROM sqlite_master
@@ -585,22 +770,15 @@ class Database:
                         SELECT name FROM fts3_shadow_tables
                       )
                       SELECT name FROM final ORDER BY 1
-                    """
-                )
-            ]
+                    """)]
         # Also hide any FTS tables that have a content= argument
-        hidden_tables += [
-            x[0]
-            for x in await self.execute(
-                """
+        hidden_tables += [x[0] for x in await self.execute("""
                   SELECT name
                   FROM sqlite_master
                   WHERE sql LIKE '%VIRTUAL TABLE%'
                     AND sql LIKE '%USING FTS%'
                     AND sql LIKE '%content=%'
-                """
-            )
-        ]
+                """)]
 
         has_spatialite = await self.execute_fn(detect_spatialite)
         if has_spatialite:
@@ -619,16 +797,11 @@ class Database:
                 "KNN",
                 "KNN2",
             ] + [
-                r[0]
-                for r in (
-                    await self.execute(
-                        """
+                r[0] for r in (await self.execute("""
                         select name from sqlite_master
                         where name like "idx_%"
                         and type = "table"
-                    """
-                    )
-                ).rows
+                    """)).rows
             ]
 
         return hidden_tables
@@ -670,6 +843,8 @@ class Database:
             tags.append("mutable")
         if self.is_memory:
             tags.append("memory")
+        if self.is_temp_disk:
+            tags.append("temp_disk")
         if self.hash:
             tags.append(f"hash={self.hash}")
         if self.size is not None:
@@ -680,17 +855,90 @@ class Database:
         return f"<Database: {self.name}{tags_str}>"
 
 
-class WriteTask:
-    __slots__ = ("fn", "task_id", "reply_queue", "isolated_connection", "transaction")
+def _apply_write_wrapper(fn, wrapper_factory, track_event):
+    """Apply a single write_wrapper context manager around fn.
 
-    def __init__(self, fn, task_id, reply_queue, isolated_connection, transaction):
+    ``wrapper_factory`` is a callable that takes ``(conn)`` and optionally
+    ``track_event``, and returns a generator that yields exactly once.
+    Code before the yield runs before ``fn(conn)``, code after the yield
+    runs after.  The result of ``fn(conn)`` is sent into the generator
+    via ``.send()``, and any exception raised by ``fn(conn)`` is thrown
+    via ``.throw()``.
+    """
+
+    def wrapped(conn):
+        gen = call_with_supported_arguments(
+            wrapper_factory, conn=conn, track_event=track_event
+        )
+        # Advance to the yield point (run "before" code)
+        try:
+            next(gen)
+        except StopIteration:
+            # Generator didn't yield — just run fn unchanged
+            return fn(conn)
+
+        # Execute the actual write
+        try:
+            result = fn(conn)
+        except Exception:
+            # Throw exception into generator so it can handle it
+            try:
+                gen.throw(*sys.exc_info())
+            except StopIteration:
+                pass
+            # Re-raise the original exception
+            raise
+        else:
+            # Send the result back through the yield
+            try:
+                gen.send(result)
+            except StopIteration:
+                pass
+            return result
+
+    return wrapped
+
+
+class WriteTask:
+    __slots__ = (
+        "fn",
+        "task_id",
+        "loop",
+        "reply_future",
+        "isolated_connection",
+        "transaction",
+    )
+
+    def __init__(
+        self, fn, task_id, loop, reply_future, isolated_connection, transaction
+    ):
         self.fn = fn
         self.task_id = task_id
-        self.reply_queue = reply_queue
+        self.loop = loop
+        self.reply_future = reply_future
         self.isolated_connection = isolated_connection
         self.transaction = transaction
 
 
+def _deliver_write_result(task, result, exception):
+    # Called from the write thread. Delivers the result back to the
+    # awaiting coroutine on its event loop via call_soon_threadsafe.
+    def _set():
+        if task.reply_future.done():
+            # Awaiter was cancelled; nothing to do.
+            return
+        if exception is not None:
+            task.reply_future.set_exception(exception)
+        else:
+            task.reply_future.set_result(result)
+
+    try:
+        task.loop.call_soon_threadsafe(_set)
+    except RuntimeError:
+        # Event loop has been closed; the awaiter is gone.
+        pass
+
+
 class QueryInterrupted(Exception):
     def __init__(self, e, sql, params):
         self.e = e

datasette/default_actions.py

@@ -48,12 +48,26 @@ def register_actions():
             resource_class=DatabaseResource,
             also_requires="view-database",
         ),
+        Action(
+            name="execute-write-sql",
+            abbr="ews",
+            description="Execute writable SQL queries",
+            resource_class=DatabaseResource,
+            also_requires="view-database",
+        ),
         Action(
             name="create-table",
             abbr="ct",
             description="Create tables",
             resource_class=DatabaseResource,
         ),
+        Action(
+            name="store-query",
+            abbr="sq",
+            description="Create stored queries",
+            resource_class=DatabaseResource,
+            also_requires="execute-sql",
+        ),
         # Table-level actions (child-level)
         Action(
             name="view-table",
@@ -85,6 +99,12 @@ def register_actions():
             description="Alter tables",
             resource_class=TableResource,
         ),
+        Action(
+            name="set-column-type",
+            abbr="sct",
+            description="Set column type",
+            resource_class=TableResource,
+        ),
         Action(
             name="drop-table",
             abbr="dt",
@@ -98,4 +118,16 @@ def register_actions():
             description="View named query results",
             resource_class=QueryResource,
         ),
+        Action(
+            name="update-query",
+            abbr="uq",
+            description="Update stored queries",
+            resource_class=QueryResource,
+        ),
+        Action(
+            name="delete-query",
+            abbr="dq",
+            description="Delete stored queries",
+            resource_class=QueryResource,
+        ),
     )

datasette/default_column_types.py

@@ -0,0 +1,81 @@
+import json
+import re
+
+import markupsafe
+
+from datasette import hookimpl
+from datasette.column_types import ColumnType, SQLiteType
+
+
+class UrlColumnType(ColumnType):
+    name = "url"
+    description = "URL"
+    sqlite_types = (SQLiteType.TEXT,)
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        if not value or not isinstance(value, str):
+            return None
+        escaped = markupsafe.escape(value.strip())
+        return markupsafe.Markup(f'<a href="{escaped}">{escaped}</a>')
+
+    async def validate(self, value, datasette):
+        if value is None or value == "":
+            return None
+        if not isinstance(value, str):
+            return "URL must be a string"
+        if not re.match(r"^https?://\S+$", value.strip()):
+            return "Invalid URL"
+        return None
+
+
+class EmailColumnType(ColumnType):
+    name = "email"
+    description = "Email address"
+    sqlite_types = (SQLiteType.TEXT,)
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        if not value or not isinstance(value, str):
+            return None
+        escaped = markupsafe.escape(value.strip())
+        return markupsafe.Markup(f'<a href="mailto:{escaped}">{escaped}</a>')
+
+    async def validate(self, value, datasette):
+        if value is None or value == "":
+            return None
+        if not isinstance(value, str):
+            return "Email must be a string"
+        if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", value.strip()):
+            return "Invalid email address"
+        return None
+
+
+class JsonColumnType(ColumnType):
+    name = "json"
+    description = "JSON data"
+    sqlite_types = (SQLiteType.TEXT,)
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        if value is None:
+            return None
+        try:
+            parsed = json.loads(value) if isinstance(value, str) else value
+            formatted = json.dumps(parsed, indent=2)
+            escaped = markupsafe.escape(formatted)
+            return markupsafe.Markup(f"<pre>{escaped}</pre>")
+        except (json.JSONDecodeError, TypeError):
+            return None
+
+    async def validate(self, value, datasette):
+        if value is None or value == "":
+            return None
+        if isinstance(value, str):
+            try:
+                json.loads(value)
+            except json.JSONDecodeError:
+                return "Invalid JSON"
+        return None
+
+
+@hookimpl
+def register_column_types(datasette):
+    return [UrlColumnType, EmailColumnType, JsonColumnType]

datasette/default_database_actions.py

@@ -0,0 +1,24 @@
+from datasette import hookimpl
+from datasette.resources import DatabaseResource
+
+
+@hookimpl
+def database_actions(datasette, actor, database, request):
+    async def inner():
+        if not datasette.get_database(database).is_mutable:
+            return []
+        if not await datasette.allowed(
+            action="execute-write-sql",
+            resource=DatabaseResource(database),
+            actor=actor,
+        ):
+            return []
+        return [
+            {
+                "href": datasette.urls.database(database) + "/-/execute-write",
+                "label": "Execute write SQL",
+                "description": "Run writable SQL with table permission checks.",
+            }
+        ]
+
+    return inner

datasette/default_debug_menu.py

@@ -0,0 +1,75 @@
+from datasette import hookimpl
+from datasette.jump import JumpSQL
+
+DEBUG_MENU_ITEMS = (
+    (
+        "/-/databases",
+        "Databases",
+        "List of databases known to this Datasette instance.",
+    ),
+    (
+        "/-/plugins",
+        "Installed plugins",
+        "Review loaded plugins, their versions and their registered hooks.",
+    ),
+    (
+        "/-/versions",
+        "Version info",
+        "Check the Python, SQLite and dependency versions used by this server.",
+    ),
+    (
+        "/-/settings",
+        "Settings",
+        "Inspect the active Datasette settings and configuration values.",
+    ),
+    (
+        "/-/permissions",
+        "Debug permissions",
+        "Test permission checks for actors, actions and resources.",
+    ),
+    (
+        "/-/messages",
+        "Debug messages",
+        "Try out temporary flash messages shown to users.",
+    ),
+    (
+        "/-/allow-debug",
+        "Debug allow rules",
+        "Explore how allow blocks match actors against permission rules.",
+    ),
+    (
+        "/-/threads",
+        "Debug threads",
+        "Inspect worker threads and database tasks.",
+    ),
+    (
+        "/-/actor",
+        "Debug actor",
+        "View the actor object for the current signed-in user.",
+    ),
+    (
+        "/-/patterns",
+        "Pattern portfolio",
+        "Browse Datasette UI patterns.",
+    ),
+)
+
+
+@hookimpl
+def jump_items_sql(datasette, actor, request):
+    async def inner():
+        if not await datasette.allowed(action="debug-menu", actor=actor):
+            return []
+
+        return [
+            JumpSQL.menu_item(
+                label=label,
+                url=datasette.urls.path(path),
+                description=description,
+                search_text=f"debug {label} {description}",
+                item_type="debug",
+            )
+            for path, label, description in DEBUG_MENU_ITEMS
+        ]
+
+    return inner

datasette/default_jump_items.py

@@ -0,0 +1,82 @@
+from datasette import hookimpl
+from datasette.jump import JumpSQL
+
+
+@hookimpl
+def jump_items_sql(datasette, actor, request):
+    async def inner():
+        database_sql, database_params = await datasette.allowed_resources_sql(
+            action="view-database", actor=actor
+        )
+        table_sql, table_params = await datasette.allowed_resources_sql(
+            action="view-table", actor=actor
+        )
+        query_sql, query_params = await datasette.allowed_resources_sql(
+            action="view-query", actor=actor
+        )
+        return [
+            JumpSQL(
+                sql=f"""
+                WITH allowed_databases AS (
+                    {database_sql}
+                )
+                SELECT
+                    'database' AS type,
+                    parent AS label,
+                    NULL AS description,
+                    json_object(
+                        'method', 'database',
+                        'database', parent
+                    ) AS url,
+                    parent AS search_text,
+                    NULL AS display_name
+                FROM allowed_databases
+                """,
+                params=database_params,
+            ),
+            JumpSQL(
+                sql=f"""
+                WITH allowed_tables AS (
+                    {table_sql}
+                )
+                SELECT
+                    CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type,
+                    allowed_tables.parent || ': ' || allowed_tables.child AS label,
+                    NULL AS description,
+                    json_object(
+                        'method', 'table',
+                        'database', allowed_tables.parent,
+                        'table', allowed_tables.child
+                    ) AS url,
+                    allowed_tables.parent || ' ' || allowed_tables.child AS search_text,
+                    NULL AS display_name
+                FROM allowed_tables
+                LEFT JOIN catalog_views
+                    ON catalog_views.database_name = allowed_tables.parent
+                   AND catalog_views.view_name = allowed_tables.child
+                """,
+                params=table_params,
+            ),
+            JumpSQL(
+                sql=f"""
+                WITH allowed_queries AS (
+                    {query_sql}
+                )
+                SELECT
+                    'query' AS type,
+                    allowed_queries.parent || ': ' || allowed_queries.child AS label,
+                    NULL AS description,
+                    json_object(
+                        'method', 'query',
+                        'database', allowed_queries.parent,
+                        'query', allowed_queries.child
+                    ) AS url,
+                    allowed_queries.parent || ' ' || allowed_queries.child AS search_text,
+                    NULL AS display_name
+                FROM allowed_queries
+                """,
+                params=query_params,
+            ),
+        ]
+
+    return inner

datasette/default_menu_links.py

@@ -1,41 +0,0 @@
-from datasette import hookimpl
-
-
-@hookimpl
-def menu_links(datasette, actor):
-    async def inner():
-        if not await datasette.allowed(action="debug-menu", actor=actor):
-            return []
-
-        return [
-            {"href": datasette.urls.path("/-/databases"), "label": "Databases"},
-            {
-                "href": datasette.urls.path("/-/plugins"),
-                "label": "Installed plugins",
-            },
-            {
-                "href": datasette.urls.path("/-/versions"),
-                "label": "Version info",
-            },
-            {
-                "href": datasette.urls.path("/-/settings"),
-                "label": "Settings",
-            },
-            {
-                "href": datasette.urls.path("/-/permissions"),
-                "label": "Debug permissions",
-            },
-            {
-                "href": datasette.urls.path("/-/messages"),
-                "label": "Debug messages",
-            },
-            {
-                "href": datasette.urls.path("/-/allow-debug"),
-                "label": "Debug allow rules",
-            },
-            {"href": datasette.urls.path("/-/threads"), "label": "Debug threads"},
-            {"href": datasette.urls.path("/-/actor"), "label": "Debug actor"},
-            {"href": datasette.urls.path("/-/patterns"), "label": "Pattern portfolio"},
-        ]
-
-    return inner

datasette/default_permissions.py

@@ -1,520 +0,0 @@
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from datasette.app import Datasette
-
-from datasette import hookimpl
-from datasette.permissions import PermissionSQL
-from datasette.utils import actor_matches_allow
-import itsdangerous
-import time
-
-
-@hookimpl(specname="permission_resources_sql")
-async def actor_restrictions_sql(datasette, actor, action):
-    """Handle actor restriction-based permission rules (_r key)."""
-    if not actor:
-        return None
-
-    restrictions = actor.get("_r") if isinstance(actor, dict) else None
-    if not restrictions:
-        return []
-
-    # Check if this action appears in restrictions (with abbreviations)
-    action_obj = datasette.actions.get(action)
-    action_checks = {action}
-    if action_obj and action_obj.abbr:
-        action_checks.add(action_obj.abbr)
-
-    # Check if this action is in the allowlist anywhere in restrictions
-    is_in_allowlist = False
-    global_actions = restrictions.get("a", [])
-    if action_checks.intersection(global_actions):
-        is_in_allowlist = True
-
-    if not is_in_allowlist:
-        for db_actions in restrictions.get("d", {}).values():
-            if action_checks.intersection(db_actions):
-                is_in_allowlist = True
-                break
-
-    if not is_in_allowlist:
-        for tables in restrictions.get("r", {}).values():
-            for table_actions in tables.values():
-                if action_checks.intersection(table_actions):
-                    is_in_allowlist = True
-                    break
-            if is_in_allowlist:
-                break
-
-    # If action not in allowlist at all, add global deny and return
-    if not is_in_allowlist:
-        sql = "SELECT NULL AS parent, NULL AS child, 0 AS allow, :actor_deny_reason AS reason"
-        return [
-            PermissionSQL(
-                sql=sql,
-                params={
-                    "actor_deny_reason": f"actor restrictions: {action} not in allowlist"
-                },
-            )
-        ]
-
-    # Action IS in allowlist - build deny + specific allows
-    selects = []
-    params = {}
-    param_counter = 0
-
-    def add_row(parent, child, allow, reason):
-        """Helper to add a parameterized SELECT statement."""
-        nonlocal param_counter
-        prefix = f"restr_{param_counter}"
-        param_counter += 1
-
-        selects.append(
-            f"SELECT :{prefix}_parent AS parent, :{prefix}_child AS child, "
-            f":{prefix}_allow AS allow, :{prefix}_reason AS reason"
-        )
-        params[f"{prefix}_parent"] = parent
-        params[f"{prefix}_child"] = child
-        params[f"{prefix}_allow"] = 1 if allow else 0
-        params[f"{prefix}_reason"] = reason
-
-    # If NOT globally allowed, add global deny as gatekeeper
-    is_globally_allowed = action_checks.intersection(global_actions)
-    if not is_globally_allowed:
-        add_row(None, None, 0, f"actor restrictions: {action} denied by default")
-    else:
-        # Globally allowed - add global allow
-        add_row(None, None, 1, f"actor restrictions: global {action}")
-
-    # Add database-level allows
-    db_restrictions = restrictions.get("d", {})
-    for db_name, db_actions in db_restrictions.items():
-        if action_checks.intersection(db_actions):
-            add_row(db_name, None, 1, f"actor restrictions: database {db_name}")
-
-    # Add resource/table-level allows
-    resource_restrictions = restrictions.get("r", {})
-    for db_name, tables in resource_restrictions.items():
-        for table_name, table_actions in tables.items():
-            if action_checks.intersection(table_actions):
-                add_row(
-                    db_name,
-                    table_name,
-                    1,
-                    f"actor restrictions: {db_name}/{table_name}",
-                )
-
-    if not selects:
-        return []
-
-    sql = "\nUNION ALL\n".join(selects)
-
-    return [PermissionSQL(sql=sql, params=params)]
-
-
-@hookimpl(specname="permission_resources_sql")
-async def root_user_permissions_sql(datasette, actor, action):
-    """Grant root user full permissions when enabled."""
-    if datasette.root_enabled and actor and actor.get("id") == "root":
-        # Add a single global-level allow rule (NULL, NULL) for root
-        # This allows root to access everything by default, but database-level
-        # and table-level deny rules in config can still block specific resources
-        return PermissionSQL.allow(reason="root user")
-    return None
-
-
-@hookimpl(specname="permission_resources_sql")
-async def config_permissions_sql(datasette, actor, action):
-    """Apply config-based permission rules from datasette.yaml."""
-    config = datasette.config or {}
-
-    def evaluate(allow_block):
-        if allow_block is None:
-            return None
-        return actor_matches_allow(actor, allow_block)
-
-    has_restrictions = actor and "_r" in actor if actor else False
-    restrictions = actor.get("_r", {}) if actor else {}
-
-    action_obj = datasette.actions.get(action)
-    action_checks = {action}
-    if action_obj and action_obj.abbr:
-        action_checks.add(action_obj.abbr)
-
-    restricted_databases: set[str] = set()
-    restricted_tables: set[tuple[str, str]] = set()
-    if has_restrictions:
-        restricted_databases = {
-            db_name
-            for db_name, db_actions in (restrictions.get("d") or {}).items()
-            if action_checks.intersection(db_actions)
-        }
-        restricted_tables = {
-            (db_name, table_name)
-            for db_name, tables in (restrictions.get("r") or {}).items()
-            for table_name, table_actions in tables.items()
-            if action_checks.intersection(table_actions)
-        }
-        # Tables implicitly reference their parent databases
-        restricted_databases.update(db for db, _ in restricted_tables)
-
-    def is_in_restriction_allowlist(parent, child, action_name):
-        """Check if a resource is in the actor's restriction allowlist for this action"""
-        if not has_restrictions:
-            return True  # No restrictions, all resources allowed
-
-        # Check global allowlist
-        if action_checks.intersection(restrictions.get("a", [])):
-            return True
-
-        # Check database-level allowlist
-        if parent and action_checks.intersection(
-            restrictions.get("d", {}).get(parent, [])
-        ):
-            return True
-
-        # Check table-level allowlist
-        if parent:
-            table_restrictions = (restrictions.get("r", {}) or {}).get(parent, {})
-            if child:
-                table_actions = table_restrictions.get(child, [])
-                if action_checks.intersection(table_actions):
-                    return True
-            else:
-                # Parent query should proceed if any child in this database is allowlisted
-                for table_actions in table_restrictions.values():
-                    if action_checks.intersection(table_actions):
-                        return True
-
-        # Parent/child both None: include if any restrictions exist for this action
-        if parent is None and child is None:
-            if action_checks.intersection(restrictions.get("a", [])):
-                return True
-            if restricted_databases:
-                return True
-            if restricted_tables:
-                return True
-
-        return False
-
-    rows = []
-
-    def add_row(parent, child, result, scope):
-        if result is None:
-            return
-        rows.append(
-            (
-                parent,
-                child,
-                bool(result),
-                f"config {'allow' if result else 'deny'} {scope}",
-            )
-        )
-
-    def add_row_allow_block(parent, child, allow_block, scope):
-        """For 'allow' blocks, always add a row if the block exists - deny if no match"""
-        if allow_block is None:
-            return
-
-        # If actor has restrictions and this resource is NOT in allowlist, skip this config rule
-        # Restrictions act as a gating filter - config cannot grant access to restricted-out resources
-        if not is_in_restriction_allowlist(parent, child, action):
-            return
-
-        result = evaluate(allow_block)
-        bool_result = bool(result)
-        # If result is None (no match) or False, treat as deny
-        rows.append(
-            (
-                parent,
-                child,
-                bool_result,  # None becomes False, False stays False, True stays True
-                f"config {'allow' if result else 'deny'} {scope}",
-            )
-        )
-        if has_restrictions and not bool_result and child is None:
-            reason = f"config deny {scope} (restriction gate)"
-            if parent is None:
-                # Root-level deny: add more specific denies for restricted resources
-                if action_obj and action_obj.takes_parent:
-                    for db_name in restricted_databases:
-                        rows.append((db_name, None, 0, reason))
-                if action_obj and action_obj.takes_child:
-                    for db_name, table_name in restricted_tables:
-                        rows.append((db_name, table_name, 0, reason))
-            else:
-                # Database-level deny: add child-level denies for restricted tables
-                if action_obj and action_obj.takes_child:
-                    for db_name, table_name in restricted_tables:
-                        if db_name == parent:
-                            rows.append((db_name, table_name, 0, reason))
-
-    root_perm = (config.get("permissions") or {}).get(action)
-    add_row(None, None, evaluate(root_perm), f"permissions for {action}")
-
-    for db_name, db_config in (config.get("databases") or {}).items():
-        db_perm = (db_config.get("permissions") or {}).get(action)
-        add_row(
-            db_name, None, evaluate(db_perm), f"permissions for {action} on {db_name}"
-        )
-
-        for table_name, table_config in (db_config.get("tables") or {}).items():
-            table_perm = (table_config.get("permissions") or {}).get(action)
-            add_row(
-                db_name,
-                table_name,
-                evaluate(table_perm),
-                f"permissions for {action} on {db_name}/{table_name}",
-            )
-
-            if action == "view-table":
-                table_allow = (table_config or {}).get("allow")
-                add_row_allow_block(
-                    db_name,
-                    table_name,
-                    table_allow,
-                    f"allow for {action} on {db_name}/{table_name}",
-                )
-
-        for query_name, query_config in (db_config.get("queries") or {}).items():
-            # query_config can be a string (just SQL) or a dict (with SQL and options)
-            if isinstance(query_config, dict):
-                query_perm = (query_config.get("permissions") or {}).get(action)
-                add_row(
-                    db_name,
-                    query_name,
-                    evaluate(query_perm),
-                    f"permissions for {action} on {db_name}/{query_name}",
-                )
-                if action == "view-query":
-                    query_allow = query_config.get("allow")
-                    add_row_allow_block(
-                        db_name,
-                        query_name,
-                        query_allow,
-                        f"allow for {action} on {db_name}/{query_name}",
-                    )
-
-        if action == "view-database":
-            db_allow = db_config.get("allow")
-            add_row_allow_block(
-                db_name, None, db_allow, f"allow for {action} on {db_name}"
-            )
-
-        if action == "execute-sql":
-            db_allow_sql = db_config.get("allow_sql")
-            add_row_allow_block(db_name, None, db_allow_sql, f"allow_sql for {db_name}")
-
-        if action == "view-table":
-            # Database-level allow block affects all tables in that database
-            db_allow = db_config.get("allow")
-            add_row_allow_block(
-                db_name, None, db_allow, f"allow for {action} on {db_name}"
-            )
-
-        if action == "view-query":
-            # Database-level allow block affects all queries in that database
-            db_allow = db_config.get("allow")
-            add_row_allow_block(
-                db_name, None, db_allow, f"allow for {action} on {db_name}"
-            )
-
-    # Root-level allow block applies to all view-* actions
-    if action == "view-instance":
-        allow_block = config.get("allow")
-        add_row_allow_block(None, None, allow_block, "allow for view-instance")
-
-    if action == "view-database":
-        # Root-level allow block also applies to view-database
-        allow_block = config.get("allow")
-        add_row_allow_block(None, None, allow_block, "allow for view-database")
-
-    if action == "view-table":
-        # Root-level allow block also applies to view-table
-        allow_block = config.get("allow")
-        add_row_allow_block(None, None, allow_block, "allow for view-table")
-
-    if action == "view-query":
-        # Root-level allow block also applies to view-query
-        allow_block = config.get("allow")
-        add_row_allow_block(None, None, allow_block, "allow for view-query")
-
-    if action == "execute-sql":
-        allow_sql = config.get("allow_sql")
-        add_row_allow_block(None, None, allow_sql, "allow_sql")
-
-    if not rows:
-        return []
-
-    parts = []
-    params = {}
-    for idx, (parent, child, allow, reason) in enumerate(rows):
-        key = f"cfg_{idx}"
-        parts.append(
-            f"SELECT :{key}_parent AS parent, :{key}_child AS child, :{key}_allow AS allow, :{key}_reason AS reason"
-        )
-        params[f"{key}_parent"] = parent
-        params[f"{key}_child"] = child
-        params[f"{key}_allow"] = 1 if allow else 0
-        params[f"{key}_reason"] = reason
-
-    sql = "\nUNION ALL\n".join(parts)
-    return [PermissionSQL(sql=sql, params=params)]
-
-
-@hookimpl(specname="permission_resources_sql")
-async def default_allow_sql_check(datasette, actor, action):
-    """Enforce default_allow_sql setting for execute-sql action."""
-    if action == "execute-sql" and not datasette.setting("default_allow_sql"):
-        return PermissionSQL.deny(reason="default_allow_sql is false")
-    return None
-
-
-@hookimpl(specname="permission_resources_sql")
-async def default_action_permissions_sql(datasette, actor, action):
-    """Apply default allow rules for standard view/execute actions."""
-    # Only apply defaults if actor has no restrictions
-    # If actor has restrictions, they've already added their own deny/allow rules
-    has_restrictions = actor and "_r" in actor
-    if has_restrictions:
-        return None
-
-    default_allow_actions = {
-        "view-instance",
-        "view-database",
-        "view-database-download",
-        "view-table",
-        "view-query",
-        "execute-sql",
-    }
-    if action in default_allow_actions:
-        reason = f"default allow for {action}".replace("'", "''")
-        return PermissionSQL.allow(reason=reason)
-
-    return None
-
-
-def restrictions_allow_action(
-    datasette: "Datasette",
-    restrictions: dict,
-    action: str,
-    resource: str | tuple[str, str],
-):
-    """
-    Check if actor restrictions allow the requested action against the requested resource.
-
-    Restrictions work on an exact-match basis: if an actor has view-table permission,
-    they can view tables, but NOT automatically view-instance or view-database.
-    Each permission is checked independently without implication logic.
-    """
-    # Does this action have an abbreviation?
-    to_check = {action}
-    action_obj = datasette.actions.get(action)
-    if action_obj and action_obj.abbr:
-        to_check.add(action_obj.abbr)
-
-    # Check if restrictions explicitly allow this action
-    # Restrictions can be at three levels:
-    # - "a": global (any resource)
-    # - "d": per-database
-    # - "r": per-table/resource
-
-    # Check global level (any resource)
-    all_allowed = restrictions.get("a")
-    if all_allowed is not None:
-        assert isinstance(all_allowed, list)
-        if to_check.intersection(all_allowed):
-            return True
-
-    # Check database level
-    if resource:
-        if isinstance(resource, str):
-            database_name = resource
-        else:
-            database_name = resource[0]
-        database_allowed = restrictions.get("d", {}).get(database_name)
-        if database_allowed is not None:
-            assert isinstance(database_allowed, list)
-            if to_check.intersection(database_allowed):
-                return True
-
-    # Check table/resource level
-    if resource is not None and not isinstance(resource, str) and len(resource) == 2:
-        database, table = resource
-        table_allowed = restrictions.get("r", {}).get(database, {}).get(table)
-        if table_allowed is not None:
-            assert isinstance(table_allowed, list)
-            if to_check.intersection(table_allowed):
-                return True
-
-    # This action is not explicitly allowed, so reject it
-    return False
-
-
-@hookimpl
-def actor_from_request(datasette, request):
-    prefix = "dstok_"
-    if not datasette.setting("allow_signed_tokens"):
-        return None
-    max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl")
-    authorization = request.headers.get("authorization")
-    if not authorization:
-        return None
-    if not authorization.startswith("Bearer "):
-        return None
-    token = authorization[len("Bearer ") :]
-    if not token.startswith(prefix):
-        return None
-    token = token[len(prefix) :]
-    try:
-        decoded = datasette.unsign(token, namespace="token")
-    except itsdangerous.BadSignature:
-        return None
-    if "t" not in decoded:
-        # Missing timestamp
-        return None
-    created = decoded["t"]
-    if not isinstance(created, int):
-        # Invalid timestamp
-        return None
-    duration = decoded.get("d")
-    if duration is not None and not isinstance(duration, int):
-        # Invalid duration
-        return None
-    if (duration is None and max_signed_tokens_ttl) or (
-        duration is not None
-        and max_signed_tokens_ttl
-        and duration > max_signed_tokens_ttl
-    ):
-        duration = max_signed_tokens_ttl
-    if duration:
-        if time.time() - created > duration:
-            # Expired
-            return None
-    actor = {"id": decoded["a"], "token": "dstok"}
-    if "_r" in decoded:
-        actor["_r"] = decoded["_r"]
-    if duration:
-        actor["token_expires"] = created + duration
-    return actor
-
-
-@hookimpl
-def skip_csrf(scope):
-    # Skip CSRF check for requests with content-type: application/json
-    if scope["type"] == "http":
-        headers = scope.get("headers") or {}
-        if dict(headers).get(b"content-type") == b"application/json":
-            return True
-
-
-@hookimpl
-def canned_queries(datasette, database, actor):
-    """Return canned queries from datasette configuration."""
-    queries = (
-        ((datasette.config or {}).get("databases") or {}).get(database) or {}
-    ).get("queries") or {}
-    return queries

datasette/default_permissions/__init__.py

@@ -0,0 +1,34 @@
+"""
+Default permission implementations for Datasette.
+
+This module provides the built-in permission checking logic through implementations
+of the permission_resources_sql hook. The hooks are organized by their purpose:
+
+1. Actor Restrictions - Enforces _r allowlists embedded in actor tokens
+2. Root User - Grants full access when --root flag is used
+3. Config Rules - Applies permissions from datasette.yaml
+4. Default Settings - Enforces default_allow_sql and default view permissions
+
+IMPORTANT: These hooks return PermissionSQL objects that are combined using SQL
+UNION/INTERSECT operations. The order of evaluation is:
+  - restriction_sql fields are INTERSECTed (all must match)
+  - Regular sql fields are UNIONed and evaluated with cascading priority
+"""
+
+from __future__ import annotations
+
+# Re-export all hooks and public utilities
+from .restrictions import (
+    actor_restrictions_sql as actor_restrictions_sql,
+    restrictions_allow_action as restrictions_allow_action,
+    ActorRestrictions as ActorRestrictions,
+)
+from .root import root_user_permissions_sql as root_user_permissions_sql
+from .config import config_permissions_sql as config_permissions_sql
+from .defaults import (
+    # Avoid "datasette.default_permissions" does not explicitly export attribute
+    default_allow_sql_check as default_allow_sql_check,
+    default_action_permissions_sql as default_action_permissions_sql,
+    default_query_permissions_sql as default_query_permissions_sql,
+    DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS,
+)

datasette/default_permissions/config.py

@@ -0,0 +1,442 @@
+"""
+Config-based permission handling for Datasette.
+
+Applies permission rules from datasette.yaml configuration.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+from datasette.utils import actor_matches_allow
+
+from .helpers import PermissionRowCollector, get_action_name_variants
+
+
+class ConfigPermissionProcessor:
+    """
+    Processes permission rules from datasette.yaml configuration.
+
+    Configuration structure:
+
+    permissions:                    # Root-level permissions block
+      view-instance:
+        id: admin
+
+    databases:
+      mydb:
+        permissions:                # Database-level permissions
+          view-database:
+            id: admin
+        allow:                      # Database-level allow block (for view-*)
+          id: viewer
+        allow_sql:                  # execute-sql allow block
+          id: analyst
+        tables:
+          users:
+            permissions:            # Table-level permissions
+              view-table:
+                id: admin
+            allow:                  # Table-level allow block
+              id: viewer
+        queries:
+          my_query:
+            permissions:            # Query-level permissions
+              view-query:
+                id: admin
+            allow:                  # Query-level allow block
+              id: viewer
+    """
+
+    def __init__(
+        self,
+        datasette: "Datasette",
+        actor: Optional[dict],
+        action: str,
+    ):
+        self.datasette = datasette
+        self.actor = actor
+        self.action = action
+        self.config = datasette.config or {}
+        self.collector = PermissionRowCollector(prefix="cfg")
+
+        # Pre-compute action variants
+        self.action_checks = get_action_name_variants(datasette, action)
+        self.action_obj = datasette.actions.get(action)
+
+        # Parse restrictions if present
+        self.has_restrictions = actor and "_r" in actor if actor else False
+        self.restrictions = actor.get("_r", {}) if actor else {}
+
+        # Pre-compute restriction info for efficiency
+        self.restricted_databases: Set[str] = set()
+        self.restricted_tables: Set[Tuple[str, str]] = set()
+
+        if self.has_restrictions:
+            self.restricted_databases = {
+                db_name
+                for db_name, db_actions in (self.restrictions.get("d") or {}).items()
+                if self.action_checks.intersection(db_actions)
+            }
+            self.restricted_tables = {
+                (db_name, table_name)
+                for db_name, tables in (self.restrictions.get("r") or {}).items()
+                for table_name, table_actions in tables.items()
+                if self.action_checks.intersection(table_actions)
+            }
+            # Tables implicitly reference their parent databases
+            self.restricted_databases.update(db for db, _ in self.restricted_tables)
+
+    def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]:
+        """Evaluate an allow block against the current actor."""
+        if allow_block is None:
+            return None
+        return actor_matches_allow(self.actor, allow_block)
+
+    def is_in_restriction_allowlist(
+        self,
+        parent: Optional[str],
+        child: Optional[str],
+    ) -> bool:
+        """Check if resource is allowed by actor restrictions."""
+        if not self.has_restrictions:
+            return True  # No restrictions, all resources allowed
+
+        # Check global allowlist
+        if self.action_checks.intersection(self.restrictions.get("a", [])):
+            return True
+
+        # Check database-level allowlist
+        if parent and self.action_checks.intersection(
+            self.restrictions.get("d", {}).get(parent, [])
+        ):
+            return True
+
+        # Check table-level allowlist
+        if parent:
+            table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {})
+            if child:
+                table_actions = table_restrictions.get(child, [])
+                if self.action_checks.intersection(table_actions):
+                    return True
+            else:
+                # Parent query should proceed if any child in this database is allowlisted
+                for table_actions in table_restrictions.values():
+                    if self.action_checks.intersection(table_actions):
+                        return True
+
+        # Parent/child both None: include if any restrictions exist for this action
+        if parent is None and child is None:
+            if self.action_checks.intersection(self.restrictions.get("a", [])):
+                return True
+            if self.restricted_databases:
+                return True
+            if self.restricted_tables:
+                return True
+
+        return False
+
+    def add_permissions_rule(
+        self,
+        parent: Optional[str],
+        child: Optional[str],
+        permissions_block: Optional[dict],
+        scope_desc: str,
+    ) -> None:
+        """Add a rule from a permissions:{action} block."""
+        if permissions_block is None:
+            return
+
+        action_allow_block = permissions_block.get(self.action)
+        result = self.evaluate_allow_block(action_allow_block)
+
+        self.collector.add(
+            parent=parent,
+            child=child,
+            allow=result,
+            reason=f"config {'allow' if result else 'deny'} {scope_desc}",
+            if_not_none=True,
+        )
+
+    def add_allow_block_rule(
+        self,
+        parent: Optional[str],
+        child: Optional[str],
+        allow_block: Any,
+        scope_desc: str,
+    ) -> None:
+        """
+        Add rules from an allow:{} block.
+
+        For allow blocks, if the block exists but doesn't match the actor,
+        this is treated as a deny. We also handle the restriction-gate logic.
+        """
+        if allow_block is None:
+            return
+
+        # Skip if resource is not in restriction allowlist
+        if not self.is_in_restriction_allowlist(parent, child):
+            return
+
+        result = self.evaluate_allow_block(allow_block)
+        bool_result = bool(result)
+
+        self.collector.add(
+            parent,
+            child,
+            bool_result,
+            f"config {'allow' if result else 'deny'} {scope_desc}",
+        )
+
+        # Handle restriction-gate: add explicit denies for restricted resources
+        self._add_restriction_gate_denies(parent, child, bool_result, scope_desc)
+
+    def _add_restriction_gate_denies(
+        self,
+        parent: Optional[str],
+        child: Optional[str],
+        is_allowed: bool,
+        scope_desc: str,
+    ) -> None:
+        """
+        When a config rule denies at a higher level, add explicit denies
+        for restricted resources to prevent child-level allows from
+        incorrectly granting access.
+        """
+        if is_allowed or child is not None or not self.has_restrictions:
+            return
+
+        if not self.action_obj:
+            return
+
+        reason = f"config deny {scope_desc} (restriction gate)"
+
+        if parent is None:
+            # Root-level deny: add denies for all restricted resources
+            if self.action_obj.takes_parent:
+                for db_name in self.restricted_databases:
+                    self.collector.add(db_name, None, False, reason)
+            if self.action_obj.takes_child:
+                for db_name, table_name in self.restricted_tables:
+                    self.collector.add(db_name, table_name, False, reason)
+        else:
+            # Database-level deny: add denies for tables in that database
+            if self.action_obj.takes_child:
+                for db_name, table_name in self.restricted_tables:
+                    if db_name == parent:
+                        self.collector.add(db_name, table_name, False, reason)
+
+    def process(self) -> Optional[PermissionSQL]:
+        """Process all config rules and return combined PermissionSQL."""
+        self._process_root_permissions()
+        self._process_databases()
+        self._process_root_allow_blocks()
+
+        return self.collector.to_permission_sql()
+
+    def _process_root_permissions(self) -> None:
+        """Process root-level permissions block."""
+        root_perms = self.config.get("permissions") or {}
+        self.add_permissions_rule(
+            None,
+            None,
+            root_perms,
+            f"permissions for {self.action}",
+        )
+
+    def _process_databases(self) -> None:
+        """Process database-level and nested configurations."""
+        databases = self.config.get("databases") or {}
+
+        for db_name, db_config in databases.items():
+            self._process_database(db_name, db_config or {})
+
+    def _process_database(self, db_name: str, db_config: dict) -> None:
+        """Process a single database's configuration."""
+        # Database-level permissions block
+        db_perms = db_config.get("permissions") or {}
+        self.add_permissions_rule(
+            db_name,
+            None,
+            db_perms,
+            f"permissions for {self.action} on {db_name}",
+        )
+
+        # Process tables
+        for table_name, table_config in (db_config.get("tables") or {}).items():
+            self._process_table(db_name, table_name, table_config or {})
+
+        # Process queries
+        for query_name, query_config in (db_config.get("queries") or {}).items():
+            self._process_query(db_name, query_name, query_config)
+
+        # Database-level allow blocks
+        self._process_database_allow_blocks(db_name, db_config)
+
+    def _process_table(
+        self,
+        db_name: str,
+        table_name: str,
+        table_config: dict,
+    ) -> None:
+        """Process a single table's configuration."""
+        # Table-level permissions block
+        table_perms = table_config.get("permissions") or {}
+        self.add_permissions_rule(
+            db_name,
+            table_name,
+            table_perms,
+            f"permissions for {self.action} on {db_name}/{table_name}",
+        )
+
+        # Table-level allow block (for view-table)
+        if self.action == "view-table":
+            self.add_allow_block_rule(
+                db_name,
+                table_name,
+                table_config.get("allow"),
+                f"allow for {self.action} on {db_name}/{table_name}",
+            )
+
+    def _process_query(
+        self,
+        db_name: str,
+        query_name: str,
+        query_config: Any,
+    ) -> None:
+        """Process a single query's configuration."""
+        # Query config can be a string (just SQL) or dict
+        if not isinstance(query_config, dict):
+            return
+
+        # Query-level permissions block
+        query_perms = query_config.get("permissions") or {}
+        self.add_permissions_rule(
+            db_name,
+            query_name,
+            query_perms,
+            f"permissions for {self.action} on {db_name}/{query_name}",
+        )
+
+        # Query-level allow block (for view-query)
+        if self.action == "view-query":
+            self.add_allow_block_rule(
+                db_name,
+                query_name,
+                query_config.get("allow"),
+                f"allow for {self.action} on {db_name}/{query_name}",
+            )
+
+    def _process_database_allow_blocks(
+        self,
+        db_name: str,
+        db_config: dict,
+    ) -> None:
+        """Process database-level allow/allow_sql blocks."""
+        # view-database allow block
+        if self.action == "view-database":
+            self.add_allow_block_rule(
+                db_name,
+                None,
+                db_config.get("allow"),
+                f"allow for {self.action} on {db_name}",
+            )
+
+        # execute-sql allow_sql block
+        if self.action == "execute-sql":
+            self.add_allow_block_rule(
+                db_name,
+                None,
+                db_config.get("allow_sql"),
+                f"allow_sql for {db_name}",
+            )
+
+        # view-table uses database-level allow for inheritance
+        if self.action == "view-table":
+            self.add_allow_block_rule(
+                db_name,
+                None,
+                db_config.get("allow"),
+                f"allow for {self.action} on {db_name}",
+            )
+
+        # view-query uses database-level allow for inheritance
+        if self.action == "view-query":
+            self.add_allow_block_rule(
+                db_name,
+                None,
+                db_config.get("allow"),
+                f"allow for {self.action} on {db_name}",
+            )
+
+    def _process_root_allow_blocks(self) -> None:
+        """Process root-level allow/allow_sql blocks."""
+        root_allow = self.config.get("allow")
+
+        if self.action == "view-instance":
+            self.add_allow_block_rule(
+                None,
+                None,
+                root_allow,
+                "allow for view-instance",
+            )
+
+        if self.action == "view-database":
+            self.add_allow_block_rule(
+                None,
+                None,
+                root_allow,
+                "allow for view-database",
+            )
+
+        if self.action == "view-table":
+            self.add_allow_block_rule(
+                None,
+                None,
+                root_allow,
+                "allow for view-table",
+            )
+
+        if self.action == "view-query":
+            self.add_allow_block_rule(
+                None,
+                None,
+                root_allow,
+                "allow for view-query",
+            )
+
+        if self.action == "execute-sql":
+            self.add_allow_block_rule(
+                None,
+                None,
+                self.config.get("allow_sql"),
+                "allow_sql",
+            )
+
+
+@hookimpl(specname="permission_resources_sql")
+async def config_permissions_sql(
+    datasette: "Datasette",
+    actor: Optional[dict],
+    action: str,
+) -> Optional[List[PermissionSQL]]:
+    """
+    Apply permission rules from datasette.yaml configuration.
+
+    This processes:
+    - permissions: blocks at root, database, table, and query levels
+    - allow: blocks for view-* actions
+    - allow_sql: blocks for execute-sql action
+    """
+    processor = ConfigPermissionProcessor(datasette, actor, action)
+    result = processor.process()
+
+    if result is None:
+        return []
+
+    return [result]

datasette/default_permissions/defaults.py

@@ -0,0 +1,114 @@
+"""
+Default permission settings for Datasette.
+
+Provides default allow rules for standard view/execute actions.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+# Actions that are allowed by default (unless --default-deny is used)
+DEFAULT_ALLOW_ACTIONS = frozenset(
+    {
+        "view-instance",
+        "view-database",
+        "view-database-download",
+        "view-table",
+        "view-query",
+        "execute-sql",
+    }
+)
+
+
+@hookimpl(specname="permission_resources_sql")
+async def default_allow_sql_check(
+    datasette: "Datasette",
+    actor: Optional[dict],
+    action: str,
+) -> Optional[PermissionSQL]:
+    """
+    Enforce the default_allow_sql setting.
+
+    When default_allow_sql is false (the default), execute-sql is denied
+    unless explicitly allowed by config or other rules.
+    """
+    if action == "execute-sql":
+        if not datasette.setting("default_allow_sql"):
+            return PermissionSQL.deny(reason="default_allow_sql is false")
+
+    return None
+
+
+@hookimpl(specname="permission_resources_sql")
+async def default_action_permissions_sql(
+    datasette: "Datasette",
+    actor: Optional[dict],
+    action: str,
+) -> Optional[PermissionSQL]:
+    """
+    Provide default allow rules for standard view/execute actions.
+
+    These defaults are skipped when datasette is started with --default-deny.
+    The restriction_sql mechanism (from actor_restrictions_sql) will still
+    filter these results if the actor has restrictions.
+    """
+    if datasette.default_deny:
+        return None
+
+    if action in DEFAULT_ALLOW_ACTIONS:
+        reason = f"default allow for {action}".replace("'", "''")
+        return PermissionSQL.allow(reason=reason)
+
+    return None
+
+
+@hookimpl(specname="permission_resources_sql")
+async def default_query_permissions_sql(
+    datasette: "Datasette",
+    actor: Optional[dict],
+    action: str,
+) -> Optional[PermissionSQL]:
+    actor_id = actor.get("id") if isinstance(actor, dict) else None
+
+    if action not in {"view-query", "update-query", "delete-query"}:
+        return None
+
+    params = {"query_owner_id": actor_id}
+    rule_sqls = []
+    if actor_id is not None:
+        if action in {"update-query", "delete-query"}:
+            # Query owner can update/delete query
+            rule_sqls.append("""
+            SELECT database_name AS parent, name AS child, 1 AS allow,
+              'query owner' AS reason
+            FROM queries
+            WHERE source = 'user'
+              AND owner_id = :query_owner_id
+            """)
+        else:
+            # Query owner can view-query
+            rule_sqls.append("""
+            SELECT database_name AS parent, name AS child, 1 AS allow,
+              'query owner' AS reason
+            FROM queries
+            WHERE owner_id = :query_owner_id
+            """)
+
+    # restriction_sql enforces private queries ONLY visible/mutable by owner
+    return PermissionSQL(
+        sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None,
+        restriction_sql="""
+        SELECT database_name AS parent, name AS child
+        FROM queries
+        WHERE is_private = 0
+           OR owner_id = :query_owner_id
+        """,
+        params=params,
+    )

datasette/default_permissions/helpers.py

@@ -0,0 +1,85 @@
+"""
+Shared helper utilities for default permission implementations.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Optional, Set
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette.permissions import PermissionSQL
+
+
+def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]:
+    """
+    Get all name variants for an action (full name and abbreviation).
+
+    Example:
+        get_action_name_variants(ds, "view-table") -> {"view-table", "vt"}
+    """
+    variants = {action}
+    action_obj = datasette.actions.get(action)
+    if action_obj and action_obj.abbr:
+        variants.add(action_obj.abbr)
+    return variants
+
+
+def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool:
+    """Check if an action (or its abbreviation) is in a list."""
+    return bool(get_action_name_variants(datasette, action).intersection(action_list))
+
+
+@dataclass
+class PermissionRow:
+    """A single permission rule row."""
+
+    parent: Optional[str]
+    child: Optional[str]
+    allow: bool
+    reason: str
+
+
+class PermissionRowCollector:
+    """Collects permission rows and converts them to PermissionSQL."""
+
+    def __init__(self, prefix: str = "row"):
+        self.rows: List[PermissionRow] = []
+        self.prefix = prefix
+
+    def add(
+        self,
+        parent: Optional[str],
+        child: Optional[str],
+        allow: Optional[bool],
+        reason: str,
+        if_not_none: bool = False,
+    ) -> None:
+        """Add a permission row. If if_not_none=True, only add if allow is not None."""
+        if if_not_none and allow is None:
+            return
+        self.rows.append(PermissionRow(parent, child, allow, reason))
+
+    def to_permission_sql(self) -> Optional[PermissionSQL]:
+        """Convert collected rows to a PermissionSQL object."""
+        if not self.rows:
+            return None
+
+        parts = []
+        params = {}
+
+        for idx, row in enumerate(self.rows):
+            key = f"{self.prefix}_{idx}"
+            parts.append(
+                f"SELECT :{key}_parent AS parent, :{key}_child AS child, "
+                f":{key}_allow AS allow, :{key}_reason AS reason"
+            )
+            params[f"{key}_parent"] = row.parent
+            params[f"{key}_child"] = row.child
+            params[f"{key}_allow"] = 1 if row.allow else 0
+            params[f"{key}_reason"] = row.reason
+
+        sql = "\nUNION ALL\n".join(parts)
+        return PermissionSQL(sql=sql, params=params)

datasette/default_permissions/restrictions.py

@@ -0,0 +1,195 @@
+"""
+Actor restriction handling for Datasette permissions.
+
+This module handles the _r (restrictions) key in actor dictionaries, which
+contains allowlists of resources the actor can access.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Optional, Set, Tuple
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+from .helpers import action_in_list, get_action_name_variants
+
+
+@dataclass
+class ActorRestrictions:
+    """Parsed actor restrictions from the _r key."""
+
+    global_actions: List[str]  # _r.a - globally allowed actions
+    database_actions: dict  # _r.d - {db_name: [actions]}
+    table_actions: dict  # _r.r - {db_name: {table: [actions]}}
+
+    @classmethod
+    def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]:
+        """Parse restrictions from actor dict. Returns None if no restrictions."""
+        if not actor:
+            return None
+        assert isinstance(actor, dict), "actor must be a dictionary"
+
+        restrictions = actor.get("_r")
+        if restrictions is None:
+            return None
+
+        return cls(
+            global_actions=restrictions.get("a", []),
+            database_actions=restrictions.get("d", {}),
+            table_actions=restrictions.get("r", {}),
+        )
+
+    def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool:
+        """Check if action is in the global allowlist."""
+        return action_in_list(datasette, action, self.global_actions)
+
+    def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]:
+        """Get database names where this action is allowed."""
+        allowed = set()
+        for db_name, db_actions in self.database_actions.items():
+            if action_in_list(datasette, action, db_actions):
+                allowed.add(db_name)
+        return allowed
+
+    def get_allowed_tables(
+        self, datasette: "Datasette", action: str
+    ) -> Set[Tuple[str, str]]:
+        """Get (database, table) pairs where this action is allowed."""
+        allowed = set()
+        for db_name, tables in self.table_actions.items():
+            for table_name, table_actions in tables.items():
+                if action_in_list(datasette, action, table_actions):
+                    allowed.add((db_name, table_name))
+        return allowed
+
+
+@hookimpl(specname="permission_resources_sql")
+async def actor_restrictions_sql(
+    datasette: "Datasette",
+    actor: Optional[dict],
+    action: str,
+) -> Optional[List[PermissionSQL]]:
+    """
+    Handle actor restriction-based permission rules.
+
+    When an actor has an "_r" key, it contains an allowlist of resources they
+    can access. This function returns restriction_sql that filters the final
+    results to only include resources in that allowlist.
+
+    The _r structure:
+    {
+        "a": ["vi", "pd"],           # Global actions allowed
+        "d": {"mydb": ["vt", "es"]}, # Database-level actions
+        "r": {"mydb": {"users": ["vt"]}}  # Table-level actions
+    }
+    """
+    if not actor:
+        return None
+
+    restrictions = ActorRestrictions.from_actor(actor)
+
+    if restrictions is None:
+        # No restrictions - all resources allowed
+        return []
+
+    # If globally allowed, no filtering needed
+    if restrictions.is_action_globally_allowed(datasette, action):
+        return []
+
+    # Build restriction SQL
+    allowed_dbs = restrictions.get_allowed_databases(datasette, action)
+    allowed_tables = restrictions.get_allowed_tables(datasette, action)
+
+    # If nothing is allowed for this action, return empty-set restriction
+    if not allowed_dbs and not allowed_tables:
+        return [
+            PermissionSQL(
+                params={"deny": f"actor restrictions: {action} not in allowlist"},
+                restriction_sql="SELECT NULL AS parent, NULL AS child WHERE 0",
+            )
+        ]
+
+    # Build UNION of allowed resources
+    selects = []
+    params = {}
+    counter = 0
+
+    # Database-level entries (parent, NULL) - allows all children
+    for db_name in allowed_dbs:
+        key = f"restr_{counter}"
+        counter += 1
+        selects.append(f"SELECT :{key}_parent AS parent, NULL AS child")
+        params[f"{key}_parent"] = db_name
+
+    # Table-level entries (parent, child)
+    for db_name, table_name in allowed_tables:
+        key = f"restr_{counter}"
+        counter += 1
+        selects.append(f"SELECT :{key}_parent AS parent, :{key}_child AS child")
+        params[f"{key}_parent"] = db_name
+        params[f"{key}_child"] = table_name
+
+    restriction_sql = "\nUNION ALL\n".join(selects)
+
+    return [PermissionSQL(params=params, restriction_sql=restriction_sql)]
+
+
+def restrictions_allow_action(
+    datasette: "Datasette",
+    restrictions: dict,
+    action: str,
+    resource: Optional[str | Tuple[str, str]],
+) -> bool:
+    """
+    Check if restrictions allow the requested action on the requested resource.
+
+    This is a synchronous utility function for use by other code that needs
+    to quickly check restriction allowlists.
+
+    Args:
+        datasette: The Datasette instance
+        restrictions: The _r dict from an actor
+        action: The action name to check
+        resource: None for global, str for database, (db, table) tuple for table
+
+    Returns:
+        True if allowed, False if denied
+    """
+    # Does this action have an abbreviation?
+    to_check = get_action_name_variants(datasette, action)
+
+    # Check global level (any resource)
+    all_allowed = restrictions.get("a")
+    if all_allowed is not None:
+        assert isinstance(all_allowed, list)
+        if to_check.intersection(all_allowed):
+            return True
+
+    # Check database level
+    if resource:
+        if isinstance(resource, str):
+            database_name = resource
+        else:
+            database_name = resource[0]
+        database_allowed = restrictions.get("d", {}).get(database_name)
+        if database_allowed is not None:
+            assert isinstance(database_allowed, list)
+            if to_check.intersection(database_allowed):
+                return True
+
+    # Check table/resource level
+    if resource is not None and not isinstance(resource, str) and len(resource) == 2:
+        database, table = resource
+        table_allowed = restrictions.get("r", {}).get(database, {}).get(table)
+        if table_allowed is not None:
+            assert isinstance(table_allowed, list)
+            if to_check.intersection(table_allowed):
+                return True
+
+    # This action is not explicitly allowed, so reject it
+    return False

datasette/default_permissions/root.py

@@ -0,0 +1,29 @@
+"""
+Root user permission handling for Datasette.
+
+Grants full permissions to the root user when --root flag is used.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+
+@hookimpl(specname="permission_resources_sql")
+async def root_user_permissions_sql(
+    datasette: "Datasette",
+    actor: Optional[dict],
+) -> Optional[PermissionSQL]:
+    """
+    Grant root user full permissions when --root flag is used.
+    """
+    if not datasette.root_enabled:
+        return None
+    if actor is not None and actor.get("id") == "root":
+        return PermissionSQL.allow(reason="root user")

datasette/default_permissions/tokens.py

@@ -0,0 +1,40 @@
+"""
+Token authentication for Datasette.
+
+Registers the default SignedTokenHandler and delegates token verification
+to datasette.verify_token() so all registered handlers are tried.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette import hookimpl
+from datasette.tokens import SignedTokenHandler
+
+
+@hookimpl
+def register_token_handler(datasette: "Datasette"):
+    """Register the default signed token handler."""
+    return SignedTokenHandler()
+
+
+@hookimpl(specname="actor_from_request")
+async def actor_from_signed_api_token(
+    datasette: "Datasette", request
+) -> Optional[dict]:
+    """
+    Authenticate requests using API tokens by delegating to all registered
+    token handlers via datasette.verify_token().
+    """
+    authorization = request.headers.get("authorization")
+    if not authorization:
+        return None
+    if not authorization.startswith("Bearer "):
+        return None
+
+    token = authorization[len("Bearer ") :]
+    return await datasette.verify_token(token)

datasette/events.py

@@ -199,6 +199,27 @@ class UpdateRowEvent(Event):
     pks: list
 
 
+@dataclass
+class RenameTableEvent(Event):
+    """
+    Event name: ``rename-table``
+
+    A table has been renamed.
+
+    :ivar database: The name of the database containing the renamed table.
+    :type database: str
+    :ivar old_table: The previous name of the table.
+    :type old_table: str
+    :ivar new_table: The new name of the table.
+    :type new_table: str
+    """
+
+    name = "rename-table"
+    database: str
+    old_table: str
+    new_table: str
+
+
 @dataclass
 class DeleteRowEvent(Event):
     """
@@ -219,6 +240,42 @@ class DeleteRowEvent(Event):
     pks: list
 
 
+@hookimpl
+def write_wrapper(datasette, database, request, transaction):
+    def wrapper(conn, track_event):
+        # Snapshot rootpage -> name before the write
+        before = {
+            row[1]: row[0]
+            for row in conn.execute(
+                "select name, rootpage from sqlite_master"
+                " where type='table' and rootpage != 0"
+            ).fetchall()
+        }
+        yield
+        # Snapshot rootpage -> name after the write
+        after = {
+            row[1]: row[0]
+            for row in conn.execute(
+                "select name, rootpage from sqlite_master"
+                " where type='table' and rootpage != 0"
+            ).fetchall()
+        }
+        # Detect renames: same rootpage, different name
+        for rootpage, old_name in before.items():
+            new_name = after.get(rootpage)
+            if new_name and new_name != old_name:
+                track_event(
+                    RenameTableEvent(
+                        actor=request.actor if request else None,
+                        database=database,
+                        old_table=old_name,
+                        new_table=new_name,
+                    )
+                )
+
+    return wrapper
+
+
 @hookimpl
 def register_events():
     return [
@@ -227,6 +284,7 @@ def register_events():
         CreateTableEvent,
         CreateTokenEvent,
         AlterTableEvent,
+        RenameTableEvent,
         DropTableEvent,
         InsertRowsEvent,
         UpsertRowsEvent,

datasette/facets.py

@@ -83,7 +83,7 @@ class Facet:
         self.ds = ds
         self.request = request
         self.database = database
-        # For foreign key expansion. Can be None for e.g. canned SQL queries:
+        # For foreign key expansion. Can be None for e.g. stored SQL queries:
         self.table = table
         self.sql = sql or f"select * from [{table}]"
         self.params = params or []
@@ -233,9 +233,7 @@ class ColumnFacet(Facet):
                 )
                 where {col} is not null
                 group by {col} order by count desc, value limit {limit}
-            """.format(
-                col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
-            )
+            """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1)
             try:
                 facet_rows_results = await self.ds.execute(
                     self.database,
@@ -482,9 +480,7 @@ class DateFacet(Facet):
                 select date({column}) from (
                     select * from ({sql}) limit 100
                 ) where {column} glob "????-??-*"
-            """.format(
-                column=escape_sqlite(column), sql=self.sql
-            )
+            """.format(column=escape_sqlite(column), sql=self.sql)
             try:
                 results = await self.ds.execute(
                     self.database,
@@ -530,9 +526,7 @@ class DateFacet(Facet):
                 )
                 where date({col}) is not null
                 group by date({col}) order by count desc, value limit {limit}
-            """.format(
-                col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
-            )
+            """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1)
             try:
                 facet_rows_results = await self.ds.execute(
                     self.database,

datasette/fixtures.py

@@ -0,0 +1,415 @@
+from datasette.utils.sqlite import sqlite3
+from datasette.utils import documented
+import itertools
+import random
+import string
+
+__all__ = [
+    "EXTRA_DATABASE_SQL",
+    "TABLES",
+    "TABLE_PARAMETERIZED_SQL",
+    "generate_compound_rows",
+    "generate_sortable_rows",
+    "populate_extra_database",
+    "populate_fixture_database",
+    "write_extra_database",
+    "write_fixture_database",
+]
+
+
+def generate_compound_rows(num):
+    """Generate rows for the compound_three_primary_keys fixture table."""
+    for a, b, c in itertools.islice(
+        itertools.product(string.ascii_lowercase, repeat=3), num
+    ):
+        yield a, b, c, f"{a}-{b}-{c}"
+
+
+def generate_sortable_rows(num):
+    """Generate rows for the sortable fixture table."""
+    rand = random.Random(42)
+    for a, b in itertools.islice(
+        itertools.product(string.ascii_lowercase, repeat=2), num
+    ):
+        yield {
+            "pk1": a,
+            "pk2": b,
+            "content": f"{a}-{b}",
+            "sortable": rand.randint(-100, 100),
+            "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]),
+            "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]),
+            "text": rand.choice(["$null", "$blah"]),
+        }
+
+
+TABLES = (
+    """
+CREATE TABLE simple_primary_key (
+  id integer primary key,
+  content text
+);
+
+CREATE TABLE primary_key_multiple_columns (
+  id varchar(30) primary key,
+  content text,
+  content2 text
+);
+
+CREATE TABLE primary_key_multiple_columns_explicit_label (
+  id varchar(30) primary key,
+  content text,
+  content2 text
+);
+
+CREATE TABLE compound_primary_key (
+  pk1 varchar(30),
+  pk2 varchar(30),
+  content text,
+  PRIMARY KEY (pk1, pk2)
+);
+
+INSERT INTO compound_primary_key VALUES ('a', 'b', 'c');
+INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c');
+INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO');
+
+CREATE TABLE compound_three_primary_keys (
+  pk1 varchar(30),
+  pk2 varchar(30),
+  pk3 varchar(30),
+  content text,
+  PRIMARY KEY (pk1, pk2, pk3)
+);
+CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content);
+
+CREATE TABLE foreign_key_references (
+  pk varchar(30) primary key,
+  foreign_key_with_label integer,
+  foreign_key_with_blank_label integer,
+  foreign_key_with_no_label varchar(30),
+  foreign_key_compound_pk1 varchar(30),
+  foreign_key_compound_pk2 varchar(30),
+  FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id),
+  FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id),
+  FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id)
+  FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2)
+);
+
+CREATE TABLE sortable (
+  pk1 varchar(30),
+  pk2 varchar(30),
+  content text,
+  sortable integer,
+  sortable_with_nulls real,
+  sortable_with_nulls_2 real,
+  text text,
+  PRIMARY KEY (pk1, pk2)
+);
+
+CREATE TABLE no_primary_key (
+  content text,
+  a text,
+  b text,
+  c text
+);
+
+CREATE TABLE [123_starts_with_digits] (
+  content text
+);
+
+CREATE VIEW paginated_view AS
+    SELECT
+        content,
+        '- ' || content || ' -' AS content_extra
+    FROM no_primary_key;
+
+CREATE TABLE "Table With Space In Name" (
+  pk varchar(30) primary key,
+  content text
+);
+
+CREATE TABLE "table/with/slashes.csv" (
+  pk varchar(30) primary key,
+  content text
+);
+
+CREATE TABLE "complex_foreign_keys" (
+  pk varchar(30) primary key,
+  f1 integer,
+  f2 integer,
+  f3 integer,
+  FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id),
+  FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id),
+  FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id)
+);
+
+CREATE TABLE "custom_foreign_key_label" (
+  pk varchar(30) primary key,
+  foreign_key_with_custom_label text,
+  FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id)
+);
+
+CREATE TABLE tags (
+    tag TEXT PRIMARY KEY
+);
+
+CREATE TABLE searchable (
+  pk integer primary key,
+  text1 text,
+  text2 text,
+  [name with . and spaces] text
+);
+
+CREATE TABLE searchable_tags (
+    searchable_id integer,
+    tag text,
+    PRIMARY KEY (searchable_id, tag),
+    FOREIGN KEY (searchable_id) REFERENCES searchable(pk),
+    FOREIGN KEY (tag) REFERENCES tags(tag)
+);
+
+INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther');
+INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma');
+
+INSERT INTO tags VALUES ("canine");
+INSERT INTO tags VALUES ("feline");
+
+INSERT INTO searchable_tags (searchable_id, tag) VALUES
+    (1, "feline"),
+    (2, "canine")
+;
+
+CREATE VIRTUAL TABLE "searchable_fts"
+    USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk");
+INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild');
+
+CREATE TABLE [select] (
+  [group] text,
+  [having] text,
+  [and] text,
+  [json] text
+);
+INSERT INTO [select] VALUES ('group', 'having', 'and',
+    '{"href": "http://example.com/", "label":"Example"}'
+);
+
+CREATE TABLE infinity (
+    value REAL
+);
+INSERT INTO infinity VALUES
+    (1e999),
+    (-1e999),
+    (1.5)
+;
+
+CREATE TABLE facet_cities (
+    id integer primary key,
+    name text
+);
+INSERT INTO facet_cities (id, name) VALUES
+    (1, 'San Francisco'),
+    (2, 'Los Angeles'),
+    (3, 'Detroit'),
+    (4, 'Memnonia')
+;
+
+CREATE TABLE facetable (
+    pk integer primary key,
+    created text,
+    planet_int integer,
+    on_earth integer,
+    state text,
+    _city_id integer,
+    _neighborhood text,
+    tags text,
+    complex_array text,
+    distinct_some_null,
+    n text,
+    FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id)
+);
+INSERT INTO facetable
+    (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n)
+VALUES
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null)
+;
+
+CREATE TABLE binary_data (
+    data BLOB
+);
+
+-- Many 2 Many demo: roadside attractions!
+
+CREATE TABLE roadside_attractions (
+    pk integer primary key,
+    name text,
+    address text,
+    url text,
+    latitude real,
+    longitude real
+);
+INSERT INTO roadside_attractions VALUES (
+    1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/",
+    37.0167, -122.0024
+);
+INSERT INTO roadside_attractions VALUES (
+    2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/",
+    37.3184, -121.9511
+);
+INSERT INTO roadside_attractions VALUES (
+    3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null,
+    37.5793, -122.3442
+);
+INSERT INTO roadside_attractions VALUES (
+    4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/",
+    37.0414, -122.0725
+);
+
+CREATE TABLE attraction_characteristic (
+    pk integer primary key,
+    name text
+);
+INSERT INTO attraction_characteristic VALUES (
+    1, "Museum"
+);
+INSERT INTO attraction_characteristic VALUES (
+    2, "Paranormal"
+);
+
+CREATE TABLE roadside_attraction_characteristics (
+    attraction_id INTEGER REFERENCES roadside_attractions(pk),
+    characteristic_id INTEGER REFERENCES attraction_characteristic(pk)
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    1, 2
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    2, 2
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    4, 2
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    3, 1
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    4, 1
+);
+
+INSERT INTO simple_primary_key VALUES (1, 'hello');
+INSERT INTO simple_primary_key VALUES (2, 'world');
+INSERT INTO simple_primary_key VALUES (3, '');
+INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO');
+INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC');
+
+INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world');
+INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2');
+
+INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b');
+INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null);
+
+INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1);
+INSERT INTO custom_foreign_key_label VALUES (1, 1);
+
+INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey');
+
+CREATE VIEW simple_view AS
+    SELECT content, upper(content) AS upper_content FROM simple_primary_key;
+
+CREATE VIEW searchable_view AS
+    SELECT * from searchable;
+
+CREATE VIEW searchable_view_configured_by_metadata AS
+    SELECT * from searchable;
+
+"""
+    + "\n".join(
+        [
+            'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format(
+                i=i + 1
+            )
+            for i in range(201)
+        ]
+    )
+    + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n'
+    + "\n".join(
+        [
+            'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format(
+                a=a, b=b, c=c, content=content
+            )
+            for a, b, c, content in generate_compound_rows(1001)
+        ]
+    )
+    + "\n".join(["""INSERT INTO sortable VALUES (
+        "{pk1}", "{pk2}", "{content}", {sortable},
+        {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}");
+    """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)])
+)
+
+TABLE_PARAMETERIZED_SQL = [
+    ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]),
+    ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]),
+    ("insert into binary_data (data) values (null);", []),
+]
+
+EXTRA_DATABASE_SQL = """
+CREATE TABLE searchable (
+  pk integer primary key,
+  text1 text,
+  text2 text
+);
+
+CREATE VIEW searchable_view AS SELECT * FROM searchable;
+
+INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog');
+INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel');
+
+CREATE VIRTUAL TABLE "searchable_fts"
+    USING FTS3 (text1, text2, content="searchable");
+INSERT INTO "searchable_fts" (rowid, text1, text2)
+    SELECT rowid, text1, text2 FROM searchable;
+"""
+
+
+@documented(label="datasette_fixtures_populate_fixture_database")
+def populate_fixture_database(conn):
+    """Populate a SQLite connection with Datasette's test fixture tables."""
+    conn.executescript(TABLES)
+    for sql, params in TABLE_PARAMETERIZED_SQL:
+        with conn:
+            conn.execute(sql, params)
+
+
+def populate_extra_database(conn):
+    """Populate a SQLite connection with the extra database used in tests."""
+    conn.executescript(EXTRA_DATABASE_SQL)
+
+
+def write_fixture_database(db_filename):
+    """Write Datasette's test fixture tables to a SQLite database file."""
+    conn = sqlite3.connect(db_filename)
+    try:
+        populate_fixture_database(conn)
+    finally:
+        conn.close()
+
+
+def write_extra_database(db_filename):
+    """Write the extra test database tables to a SQLite database file."""
+    conn = sqlite3.connect(db_filename)
+    try:
+        populate_extra_database(conn)
+    finally:
+        conn.close()

datasette/hookspecs.py

@@ -55,7 +55,17 @@ def publish_subcommand(publish):
 
 
 @hookspec
-def render_cell(row, value, column, table, database, datasette, request):
+def render_cell(
+    row,
+    value,
+    column,
+    table,
+    pks,
+    database,
+    datasette,
+    request,
+    column_type,
+):
     """Customize rendering of HTML table cell values"""
 
 
@@ -74,6 +84,11 @@ def register_actions(datasette):
     """Register actions: returns a list of datasette.permission.Action objects"""
 
 
+@hookspec
+def register_column_types(datasette):
+    """Return a list of ColumnType subclasses"""
+
+
 @hookspec
 def register_routes(datasette):
     """Register URL routes: return a list of (regex, view_function) pairs"""
@@ -122,11 +137,6 @@ def permission_resources_sql(datasette, actor, action):
     """
 
 
-@hookspec
-def canned_queries(datasette, database, actor):
-    """Return a dictionary of canned query definitions or an awaitable function that returns them"""
-
-
 @hookspec
 def register_magic_parameters(datasette):
     """Return a list of (name, function) magic parameter functions"""
@@ -142,6 +152,11 @@ def menu_links(datasette, actor, request):
     """Links for the navigation menu"""
 
 
+@hookspec
+def jump_items_sql(datasette, actor, request):
+    """SQL fragments for extra items in the jump menu"""
+
+
 @hookspec
 def row_actions(datasette, actor, request, database, table, row):
     """Links for the row actions menu"""
@@ -159,7 +174,7 @@ def view_actions(datasette, actor, database, view, request):
 
 @hookspec
 def query_actions(datasette, actor, database, query_name, request, sql, params):
-    """Links for the query and canned query actions menu"""
+    """Links for the query and stored query actions menu"""
 
 
 @hookspec
@@ -172,11 +187,6 @@ def homepage_actions(datasette, actor, request):
     """Links for the homepage actions menu"""
 
 
-@hookspec
-def skip_csrf(datasette, scope):
-    """Mechanism for skipping CSRF checks for certain requests"""
-
-
 @hookspec
 def handle_exception(datasette, request, exception):
     """Handle an uncaught exception. Can return a Response or None."""
@@ -218,5 +228,38 @@ def top_query(datasette, request, database, sql):
 
 
 @hookspec
-def top_canned_query(datasette, request, database, query_name):
-    """HTML to include at the top of the canned query page"""
+def top_stored_query(datasette, request, database, query_name):
+    """HTML to include at the top of the stored query page"""
+
+
+@hookspec
+def register_token_handler(datasette):
+    """Return a TokenHandler instance for token creation and verification"""
+
+
+@hookspec
+def write_wrapper(datasette, database, request, transaction):
+    """Called when a write function is about to execute.
+
+    Return a generator function that accepts a ``conn`` argument and
+    optionally a ``track_event`` argument.  The generator should
+    ``yield`` exactly once: code before the ``yield`` runs before
+    the write, code after the ``yield`` runs after the write
+    completes. The result of the write is sent back through the
+    ``yield``, so you can capture it with ``result = yield``.
+
+    If your generator accepts ``track_event``, you can call
+    ``track_event(event)`` to queue an event that will be dispatched
+    via ``datasette.track_event()`` after the write commits
+    successfully.  Events are discarded if the write raises an
+    exception.
+
+    If the write raises an exception, it is thrown into the generator
+    so you can handle it with a try/except around the ``yield``.
+
+    ``request`` may be ``None`` for writes not originating from an
+    HTTP request.  ``transaction`` is ``True`` if the write will
+    be wrapped in a transaction.
+
+    Return ``None`` to skip wrapping.
+    """

datasette/inspect.py

@@ -10,7 +10,6 @@ from .utils import (
     sqlite3,
 )
 
-
 HASH_BLOCK_SIZE = 1024 * 1024
 
 
@@ -70,16 +69,11 @@ def inspect_tables(conn, database_metadata):
         tables[table]["foreign_keys"] = info
 
     # Mark tables 'hidden' if they relate to FTS virtual tables
-    hidden_tables = [
-        r["name"]
-        for r in conn.execute(
-            """
+    hidden_tables = [r["name"] for r in conn.execute("""
                 select name from sqlite_master
                 where rootpage = 0
                 and sql like '%VIRTUAL TABLE%USING FTS%'
-            """
-        )
-    ]
+            """)]
 
     if detect_spatialite(conn):
         # Also hide Spatialite internal tables
@@ -94,14 +88,11 @@ def inspect_tables(conn, database_metadata):
             "views_geometry_columns",
             "virts_geometry_columns",
         ] + [
-            r["name"]
-            for r in conn.execute(
-                """
+            r["name"] for r in conn.execute("""
                     select name from sqlite_master
                     where name like "idx_%"
                     and type = "table"
-                """
-            )
+                """)
         ]
 
     for t in tables.keys():

datasette/jump.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class JumpSQL:
+    sql: str
+    params: dict[str, Any] | None = None
+    database: str | None = None
+
+    @classmethod
+    def menu_item(
+        cls,
+        *,
+        label: str,
+        url: str,
+        description: str = "Menu item",
+        search_text: str | None = None,
+        display_name: str | None = None,
+        item_type: str = "menu",
+    ) -> "JumpSQL":
+        if search_text is None:
+            search_text = " ".join(
+                text for text in (label, display_name, description) if text is not None
+            )
+        return cls(
+            sql="""
+            SELECT
+                :type AS type,
+                :label AS label,
+                :description AS description,
+                :url AS url,
+                :search_text AS search_text,
+                :display_name AS display_name
+            """,
+            params={
+                "type": item_type,
+                "label": label,
+                "description": description,
+                "url": url,
+                "search_text": search_text,
+                "display_name": display_name,
+            },
+        )
+
+
+_PARAM_RE = re.compile(r"(?<!:):([A-Za-z_][A-Za-z0-9_]*)")
+
+
+def namespace_sql_params(sql: str, params: dict[str, Any], prefix: str):
+    """Rename named SQL parameters so UNION query parameters cannot collide."""
+    if not params:
+        return sql, {}
+
+    renamed = {key: f"{prefix}_{key}" for key in params}
+
+    def replace(match):
+        key = match.group(1)
+        if key not in renamed:
+            return match.group(0)
+        return f":{renamed[key]}"
+
+    return _PARAM_RE.sub(replace, sql), {
+        renamed[key]: value for key, value in params.items()
+    }

datasette/permissions.py

@@ -1,6 +1,32 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Any, NamedTuple
+import contextvars
+
+# Context variable to track when permission checks should be skipped
+_skip_permission_checks = contextvars.ContextVar(
+    "skip_permission_checks", default=False
+)
+
+
+class SkipPermissions:
+    """Context manager to temporarily skip permission checks.
+
+    This is not a stable API and may change in future releases.
+
+    Usage:
+        with SkipPermissions():
+            # Permission checks are skipped within this block
+            response = await datasette.client.get("/protected")
+    """
+
+    def __enter__(self):
+        self.token = _skip_permission_checks.set(True)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        _skip_permission_checks.reset(self.token)
+        return False
 
 
 class Resource(ABC):
@@ -79,7 +105,7 @@ class Resource(ABC):
 
     @classmethod
     @abstractmethod
-    def resources_sql(cls) -> str:
+    async def resources_sql(cls, datasette, actor=None) -> str:
         """
         Return SQL query that returns all resources of this type.
 
@@ -138,13 +164,20 @@ class PermissionSQL:
       child  TEXT NULL,
       allow  INTEGER,    -- 1 allow, 0 deny
       reason TEXT
+
+    For restriction-only plugins, sql can be None and only restriction_sql is provided.
     """
 
-    sql: str  # SQL that SELECTs the 4 columns above
+    sql: str | None = (
+        None  # SQL that SELECTs the 4 columns above (can be None for restriction-only)
+    )
     params: dict[str, Any] | None = (
         None  # bound params for the SQL (values only; no ':' prefix)
     )
     source: str | None = None  # System will set this to the plugin name
+    restriction_sql: str | None = (
+        None  # Optional SQL that returns (parent, child) for restriction filtering
+    )
 
     @classmethod
     def allow(cls, reason: str, _allow: bool = True) -> "PermissionSQL":

datasette/plugins.py

@@ -23,10 +23,14 @@ DEFAULT_PLUGINS = (
     "datasette.sql_functions",
     "datasette.actor_auth_cookie",
     "datasette.default_permissions",
+    "datasette.default_permissions.tokens",
     "datasette.default_actions",
+    "datasette.default_column_types",
     "datasette.default_magic_parameters",
     "datasette.blob_renderer",
-    "datasette.default_menu_links",
+    "datasette.default_debug_menu",
+    "datasette.default_jump_items",
+    "datasette.default_database_actions",
     "datasette.handle_exception",
     "datasette.forbidden",
     "datasette.events",
@@ -94,21 +98,24 @@ def get_plugins():
     for plugin in pm.get_plugins():
         static_path = None
         templates_path = None
-        if plugin.__name__ not in DEFAULT_PLUGINS:
+        plugin_name = (
+            plugin.__name__
+            if hasattr(plugin, "__name__")
+            else plugin.__class__.__name__
+        )
+        if plugin_name not in DEFAULT_PLUGINS:
             try:
-                if (importlib_resources.files(plugin.__name__) / "static").is_dir():
-                    static_path = str(
-                        importlib_resources.files(plugin.__name__) / "static"
-                    )
-                if (importlib_resources.files(plugin.__name__) / "templates").is_dir():
+                if (importlib_resources.files(plugin_name) / "static").is_dir():
+                    static_path = str(importlib_resources.files(plugin_name) / "static")
+                if (importlib_resources.files(plugin_name) / "templates").is_dir():
                     templates_path = str(
-                        importlib_resources.files(plugin.__name__) / "templates"
+                        importlib_resources.files(plugin_name) / "templates"
                     )
             except (TypeError, ModuleNotFoundError):
                 # Caused by --plugins_dir= plugins
                 pass
         plugin_info = {
-            "name": plugin.__name__,
+            "name": plugin_name,
             "static_path": static_path,
             "templates_path": templates_path,
             "hooks": [h.name for h in pm.get_hookcallers(plugin)],

datasette/publish/cloudrun.py

@@ -3,7 +3,7 @@ import click
 import json
 import os
 import re
-from subprocess import check_call, check_output
+from subprocess import CalledProcessError, check_call, check_output
 
 from .common import (
     add_common_publish_arguments_and_options,
@@ -23,7 +23,9 @@ def publish_subcommand(publish):
         help="Application name to use when building",
     )
     @click.option(
-        "--service", default="", help="Cloud Run service to deploy (or over-write)"
+        "--service",
+        default="",
+        help="Cloud Run service to deploy (or over-write)",
     )
     @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension")
     @click.option(
@@ -55,13 +57,32 @@ def publish_subcommand(publish):
     @click.option(
         "--max-instances",
         type=int,
-        help="Maximum Cloud Run instances",
+        default=1,
+        show_default=True,
+        help="Maximum Cloud Run instances (use 0 to remove the limit)",
     )
     @click.option(
         "--min-instances",
         type=int,
         help="Minimum Cloud Run instances",
     )
+    @click.option(
+        "--artifact-repository",
+        default="datasette",
+        show_default=True,
+        help="Artifact Registry repository to store the image",
+    )
+    @click.option(
+        "--artifact-region",
+        default="us",
+        show_default=True,
+        help="Artifact Registry location (region or multi-region)",
+    )
+    @click.option(
+        "--artifact-project",
+        default=None,
+        help="Project ID for Artifact Registry (defaults to the active project)",
+    )
     def cloudrun(
         files,
         metadata,
@@ -91,6 +112,9 @@ def publish_subcommand(publish):
         apt_get_extras,
         max_instances,
         min_instances,
+        artifact_repository,
+        artifact_region,
+        artifact_project,
     ):
         "Publish databases to Datasette running on Cloud Run"
         fail_if_publish_binary_not_installed(
@@ -100,6 +124,21 @@ def publish_subcommand(publish):
             "gcloud config get-value project", shell=True, universal_newlines=True
         ).strip()
 
+        artifact_project = artifact_project or project
+
+        # Ensure Artifact Registry exists for the target image
+        _ensure_artifact_registry(
+            artifact_project=artifact_project,
+            artifact_region=artifact_region,
+            artifact_repository=artifact_repository,
+        )
+
+        artifact_host = (
+            artifact_region
+            if artifact_region.endswith("-docker.pkg.dev")
+            else f"{artifact_region}-docker.pkg.dev"
+        )
+
         if not service:
             # Show the user their current services, then prompt for one
             click.echo("Please provide a service name for this deployment\n")
@@ -117,6 +156,11 @@ def publish_subcommand(publish):
                 click.echo("")
             service = click.prompt("Service name", type=str)
 
+        image_id = (
+            f"{artifact_host}/{artifact_project}/"
+            f"{artifact_repository}/datasette-{service}"
+        )
+
         extra_metadata = {
             "title": title,
             "license": license,
@@ -173,7 +217,6 @@ def publish_subcommand(publish):
                     print(fp.read())
                 print("\n====================\n")
 
-            image_id = f"gcr.io/{project}/datasette-{service}"
             check_call(
                 "gcloud builds submit --tag {}{}".format(
                     image_id, " --timeout {}".format(timeout) if timeout else ""
@@ -187,7 +230,7 @@ def publish_subcommand(publish):
             ("--max-instances", max_instances),
             ("--min-instances", min_instances),
         ):
-            if value:
+            if value is not None:
                 extra_deploy_options.append("{} {}".format(option, value))
         check_call(
             "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format(
@@ -199,6 +242,52 @@ def publish_subcommand(publish):
         )
 
 
+def _ensure_artifact_registry(artifact_project, artifact_region, artifact_repository):
+    """Ensure Artifact Registry API is enabled and the repository exists."""
+
+    enable_cmd = (
+        "gcloud services enable artifactregistry.googleapis.com "
+        f"--project {artifact_project} --quiet"
+    )
+    try:
+        check_call(enable_cmd, shell=True)
+    except CalledProcessError as exc:
+        raise click.ClickException(
+            "Failed to enable artifactregistry.googleapis.com. "
+            "Please ensure you have permissions to manage services."
+        ) from exc
+
+    describe_cmd = (
+        "gcloud artifacts repositories describe {repo} --project {project} "
+        "--location {location} --quiet"
+    ).format(
+        repo=artifact_repository,
+        project=artifact_project,
+        location=artifact_region,
+    )
+    try:
+        check_call(describe_cmd, shell=True)
+        return
+    except CalledProcessError:
+        create_cmd = (
+            "gcloud artifacts repositories create {repo} --repository-format=docker "
+            '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet'
+        ).format(
+            repo=artifact_repository,
+            location=artifact_region,
+            project=artifact_project,
+        )
+        try:
+            check_call(create_cmd, shell=True)
+            click.echo(f"Created Artifact Registry repository '{artifact_repository}'")
+        except CalledProcessError as exc:
+            raise click.ClickException(
+                "Failed to create Artifact Registry repository. "
+                "Use --artifact-repository/--artifact-region to point to an existing repo "
+                "or create one manually."
+            ) from exc
+
+
 def get_existing_services():
     services = json.loads(
         check_output(
@@ -214,6 +303,7 @@ def get_existing_services():
             "url": service["status"]["address"]["url"],
         }
         for service in services
+        if "url" in service["status"]
     ]
 
 

datasette/resources.py

@@ -13,7 +13,7 @@ class DatabaseResource(Resource):
         super().__init__(parent=database, child=None)
 
     @classmethod
-    async def resources_sql(cls, datasette) -> str:
+    async def resources_sql(cls, datasette, actor=None) -> str:
         return """
             SELECT database_name AS parent, NULL AS child
             FROM catalog_databases
@@ -30,7 +30,7 @@ class TableResource(Resource):
         super().__init__(parent=database, child=table)
 
     @classmethod
-    async def resources_sql(cls, datasette) -> str:
+    async def resources_sql(cls, datasette, actor=None) -> str:
         return """
             SELECT database_name AS parent, table_name AS child
             FROM catalog_tables
@@ -41,7 +41,7 @@ class TableResource(Resource):
 
 
 class QueryResource(Resource):
-    """A canned query in a database."""
+    """A stored query in a database."""
 
     name = "query"
     parent_class = DatabaseResource
@@ -50,41 +50,9 @@ class QueryResource(Resource):
         super().__init__(parent=database, child=query)
 
     @classmethod
-    async def resources_sql(cls, datasette) -> str:
-        from datasette.plugins import pm
-        from datasette.utils import await_me_maybe
-
-        # Get all databases from catalog
-        db = datasette.get_internal_database()
-        result = await db.execute("SELECT database_name FROM catalog_databases")
-        databases = [row[0] for row in result.rows]
-
-        # Gather all canned queries from all databases
-        query_pairs = []
-        for database_name in databases:
-            # Call the hook to get queries (including from config via default plugin)
-            for queries_result in pm.hook.canned_queries(
-                datasette=datasette,
-                database=database_name,
-                actor=None,  # Get ALL queries for resource enumeration
-            ):
-                queries = await await_me_maybe(queries_result)
-                if queries:
-                    for query_name in queries.keys():
-                        query_pairs.append((database_name, query_name))
-
-        # Build SQL
-        if not query_pairs:
-            return "SELECT NULL AS parent, NULL AS child WHERE 0"
-
-        # Generate UNION ALL query
-        selects = []
-        for db_name, query_name in query_pairs:
-            # Escape single quotes by doubling them
-            db_escaped = db_name.replace("'", "''")
-            query_escaped = query_name.replace("'", "''")
-            selects.append(
-                f"SELECT '{db_escaped}' AS parent, '{query_escaped}' AS child"
-            )
-
-        return " UNION ALL ".join(selects)
+    async def resources_sql(cls, datasette, actor=None) -> str:
+        return """
+            SELECT q.database_name AS parent, q.name AS child
+            FROM queries q
+            JOIN catalog_databases cd ON cd.database_name = q.database_name
+        """

datasette/static/app.css

@@ -63,6 +63,14 @@ em {
 }
 /* end reset */
 
+/* Modal CSS variables (shared by web components via Shadow DOM) */
+:root {
+    --modal-backdrop-bg: rgba(0, 0, 0, 0.5);
+    --modal-backdrop-blur: blur(4px);
+    --modal-border-radius: 0.75rem;
+    --modal-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04);
+    --modal-animation-duration: 0.2s;
+}
 
 body {
     margin: 0;
@@ -354,6 +362,32 @@ form.nav-menu-logout {
 .nav-menu-inner a {
     display: block;
 }
+.nav-menu-inner button.button-as-link {
+    display: block;
+    width: 100%;
+    text-align: left;
+    font: inherit;
+}
+.nav-menu-inner .keyboard-shortcut {
+    float: right;
+    box-sizing: border-box;
+    min-width: 1.4em;
+    margin-left: 0.75rem;
+    padding: 0 0.35em;
+    border: 1px solid rgba(255,255,244,0.6);
+    border-radius: 3px;
+    background: rgba(255,255,244,0.12);
+    font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
+    font-size: 0.85em;
+    line-height: 1.35;
+    text-align: center;
+    text-decoration: none;
+}
+@media (max-width: 640px) {
+    .nav-menu-inner .keyboard-shortcut {
+        display: none;
+    }
+}
 
 /* Table/database actions menu */
 .page-action-menu {
@@ -647,10 +681,14 @@ button.core[type=button] {
     border-radius: 3px;
     -webkit-appearance: none;
     padding: 9px 4px;
-    font-size: 1em;
+    font-size: 16px;
     font-family: Helvetica, sans-serif;
 }
 
+#_search {
+    font-size: 16px;
+}
+
 
 
 
@@ -730,6 +768,474 @@ p.zero-results {
 .select-wrapper.small-screen-only {
     display: none;
 }
+
+@keyframes datasette-modal-slide-in {
+    from {
+        opacity: 0;
+        transform: translateY(-20px) scale(0.95);
+    }
+    to {
+        opacity: 1;
+        transform: translateY(0) scale(1);
+    }
+}
+
+@keyframes datasette-modal-fade-in {
+    from { opacity: 0; }
+    to { opacity: 1; }
+}
+
+dialog.mobile-column-actions-dialog {
+    --ink: #0f0f0f;
+    --paper: #f5f3ef;
+    --muted: #6b6b6b;
+    --rule: #e2dfd8;
+    --accent: #1a56db;
+    --card: #ffffff;
+    border: none;
+    border-radius: var(--modal-border-radius, 0.75rem);
+    padding: 0;
+    margin: auto;
+    width: min(420px, calc(100vw - 32px));
+    max-width: 95vw;
+    max-height: min(640px, calc(100vh - 32px));
+    box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
+    animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out;
+    overflow: hidden;
+    font-family: system-ui, -apple-system, sans-serif;
+    background: var(--card);
+}
+
+dialog.mobile-column-actions-dialog[open] {
+    display: flex;
+    flex-direction: column;
+}
+
+dialog.mobile-column-actions-dialog::backdrop {
+    background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
+    backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+    -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+    animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out;
+}
+
+.mobile-column-actions-dialog .modal-header {
+    padding: 20px 24px 16px;
+    border-bottom: 1px solid var(--rule);
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    gap: 12px;
+    flex-shrink: 0;
+}
+
+.mobile-column-actions-dialog .modal-title {
+    font-size: 1rem;
+    font-weight: 600;
+    color: var(--ink);
+}
+
+.mobile-column-actions-dialog .modal-meta {
+    font-family: ui-monospace, monospace;
+    font-size: 0.7rem;
+    color: var(--muted);
+    background: var(--paper);
+    padding: 3px 9px;
+    border-radius: 20px;
+}
+
+.mobile-column-actions-dialog .list-wrap {
+    flex: 1 1 auto;
+    min-height: 0;
+    overflow-y: auto;
+    overflow-x: hidden;
+    position: relative;
+    overscroll-behavior: contain;
+    -webkit-overflow-scrolling: touch;
+}
+
+.mobile-column-actions-dialog .list-wrap::before,
+.mobile-column-actions-dialog .list-wrap::after {
+    content: "";
+    position: sticky;
+    display: block;
+    left: 0;
+    right: 0;
+    height: 20px;
+    pointer-events: none;
+    z-index: 5;
+}
+
+.mobile-column-actions-dialog .list-wrap::before {
+    top: 0;
+    background: linear-gradient(to bottom, rgba(255,255,255,0.9), transparent);
+}
+
+.mobile-column-actions-dialog .list-wrap::after {
+    bottom: 0;
+    background: linear-gradient(to top, rgba(255,255,255,0.9), transparent);
+    margin-top: -20px;
+}
+
+.mobile-column-top-actions {
+    padding: 10px 24px 0;
+}
+
+.mobile-column-top-action {
+    display: inline-block;
+    text-decoration: none;
+}
+
+.mobile-column-section {
+    border-bottom: 1px solid var(--rule);
+}
+
+.mobile-column-actions-dialog .col-header {
+    width: 100%;
+    padding: 12px 24px;
+    font: inherit;
+    font-weight: 600;
+    border: 0;
+    background: none;
+    cursor: pointer;
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    text-align: left;
+}
+
+.mobile-column-header-text {
+    display: flex;
+    flex-direction: column;
+    gap: 0.15rem;
+}
+
+.mobile-column-name {
+    color: var(--ink);
+}
+
+.mobile-column-meta {
+    color: var(--muted);
+    font-size: 0.78em;
+    font-family: ui-monospace, monospace;
+    font-weight: normal;
+}
+
+.mobile-column-chevron {
+    color: var(--muted);
+    transition: transform 0.2s ease-out;
+}
+
+.mobile-column-actions-dialog .col-header[aria-expanded="true"] .mobile-column-chevron {
+    transform: rotate(180deg);
+}
+
+.mobile-column-actions-dialog .col-actions[hidden] {
+    display: none;
+}
+
+.mobile-column-actions-dialog .col-actions ul,
+.mobile-column-actions-dialog .col-actions li {
+    margin: 0;
+    padding: 0;
+    list-style-type: none;
+}
+
+.mobile-column-actions-dialog .col-actions a,
+.mobile-column-actions-dialog .col-actions button {
+    display: block;
+    width: 100%;
+    padding: 10px 24px 10px 40px;
+    color: var(--ink);
+    text-align: left;
+    font: inherit;
+    text-decoration: none;
+    background: none;
+    border: 0;
+    border-top: 1px solid #f5f5f5;
+    cursor: pointer;
+}
+
+.mobile-column-actions-dialog .col-actions a:hover,
+.mobile-column-actions-dialog .col-actions button:hover {
+    background: var(--paper);
+}
+
+.mobile-column-actions-dialog .col-actions a:active,
+.mobile-column-actions-dialog .col-actions button:active {
+    background: #eee;
+}
+
+.mobile-column-description,
+.mobile-column-no-actions {
+    margin: 0;
+    padding: 0 24px 12px 24px;
+    color: var(--muted);
+    font-size: 0.85em;
+}
+
+.mobile-column-actions-dialog .modal-footer {
+    padding: 14px 20px;
+    border-top: 1px solid var(--rule);
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    flex-shrink: 0;
+    background: var(--paper);
+}
+
+.mobile-column-actions-dialog .footer-info {
+    flex: 1;
+    font-family: ui-monospace, monospace;
+    font-size: 0.68rem;
+    color: var(--muted);
+}
+
+.mobile-column-actions-dialog .btn {
+    border: none;
+    border-radius: 5px;
+    padding: 9px 20px;
+    font-size: 0.85rem;
+    font-weight: 500;
+    cursor: pointer;
+    touch-action: manipulation;
+    font-family: inherit;
+    transition: background 0.12s;
+}
+
+.mobile-column-actions-dialog .btn-ghost {
+    background: transparent;
+    color: var(--muted);
+    border: 1px solid var(--rule);
+}
+
+.mobile-column-actions-dialog .btn-ghost:hover {
+    background: var(--rule);
+    color: var(--ink);
+}
+
+dialog.set-column-type-dialog {
+    --ink: #0f0f0f;
+    --paper: #f5f3ef;
+    --muted: #6b6b6b;
+    --rule: #e2dfd8;
+    --accent: #1a56db;
+    --card: #ffffff;
+    border: none;
+    border-radius: var(--modal-border-radius, 0.75rem);
+    padding: 0;
+    margin: auto;
+    width: min(520px, calc(100vw - 32px));
+    max-width: 95vw;
+    max-height: min(720px, calc(100vh - 32px));
+    box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
+    animation: datasette-modal-slide-in var(--modal-animation-duration, 0.2s) ease-out;
+    overflow: hidden;
+    font-family: system-ui, -apple-system, sans-serif;
+    background: var(--card);
+}
+
+dialog.set-column-type-dialog[open] {
+    display: flex;
+    flex-direction: column;
+}
+
+dialog.set-column-type-dialog::backdrop {
+    background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
+    backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+    -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+    animation: datasette-modal-fade-in var(--modal-animation-duration, 0.2s) ease-out;
+}
+
+.set-column-type-dialog .modal-header {
+    padding: 20px 24px 12px;
+    border-bottom: 1px solid var(--rule);
+    display: flex;
+    align-items: center;
+    justify-content: space-between;
+    gap: 12px;
+    flex-shrink: 0;
+}
+
+.set-column-type-dialog .modal-title {
+    font-size: 1rem;
+    font-weight: 600;
+    color: var(--ink);
+}
+
+.set-column-type-dialog .modal-meta {
+    font-family: ui-monospace, monospace;
+    font-size: 0.7rem;
+    color: var(--muted);
+    background: var(--paper);
+    padding: 3px 9px;
+    border-radius: 20px;
+}
+
+.set-column-type-status,
+.set-column-type-empty,
+.set-column-type-error {
+    margin: 0;
+    padding: 12px 24px 0;
+}
+
+.set-column-type-status,
+.set-column-type-empty {
+    color: var(--muted);
+    font-size: 0.9rem;
+}
+
+.set-column-type-error {
+    color: #b91c1c;
+    font-size: 0.9rem;
+}
+
+.set-column-type-options {
+    padding: 16px 24px 24px;
+    overflow-y: auto;
+    display: grid;
+    gap: 12px;
+}
+
+.set-column-type-option {
+    display: grid;
+    grid-template-columns: auto 1fr;
+    gap: 12px;
+    align-items: start;
+    padding: 14px 16px;
+    border: 1px solid var(--rule);
+    border-radius: 8px;
+    background: #fcfbf9;
+    cursor: pointer;
+}
+
+.set-column-type-option:focus-within {
+    border-color: var(--accent);
+    box-shadow: 0 0 0 3px rgba(26, 86, 219, 0.12);
+}
+
+.set-column-type-option input {
+    margin-top: 3px;
+}
+
+.set-column-type-option-content {
+    display: grid;
+    gap: 4px;
+}
+
+.set-column-type-option-name {
+    font-family: ui-monospace, monospace;
+    font-size: 0.95rem;
+    color: var(--ink);
+}
+
+.set-column-type-option-description {
+    color: var(--muted);
+    font-size: 0.9rem;
+}
+
+.set-column-type-dialog .modal-footer {
+    padding: 14px 20px;
+    border-top: 1px solid var(--rule);
+    display: flex;
+    align-items: center;
+    gap: 10px;
+    flex-shrink: 0;
+    background: var(--paper);
+}
+
+.set-column-type-dialog .footer-info {
+    flex: 1;
+    font-family: ui-monospace, monospace;
+    font-size: 0.68rem;
+    color: var(--muted);
+}
+
+.set-column-type-dialog .btn {
+    border: none;
+    border-radius: 5px;
+    padding: 9px 20px;
+    font-size: 0.85rem;
+    font-weight: 500;
+    cursor: pointer;
+    touch-action: manipulation;
+    font-family: inherit;
+    transition: background 0.12s;
+}
+
+.set-column-type-dialog .btn-ghost {
+    background: transparent;
+    color: var(--muted);
+    border: 1px solid var(--rule);
+}
+
+.set-column-type-dialog .btn-ghost:hover {
+    background: var(--rule);
+    color: var(--ink);
+}
+
+.set-column-type-dialog .btn-primary {
+    background: var(--accent);
+    color: #fff;
+}
+
+.set-column-type-dialog .btn-primary:hover {
+    background: #1949b8;
+}
+
+.set-column-type-dialog .btn:disabled {
+    opacity: 0.65;
+    cursor: wait;
+}
+
+@media (max-width: 640px) {
+    dialog.mobile-column-actions-dialog {
+        width: 95vw;
+        max-height: 85vh;
+        border-radius: 0.5rem;
+    }
+
+    .mobile-column-actions-dialog .modal-header {
+        padding: 16px 18px 14px;
+    }
+
+    .mobile-column-top-actions {
+        padding-left: 18px;
+        padding-right: 18px;
+    }
+
+    .mobile-column-actions-dialog .col-header {
+        padding-left: 18px;
+        padding-right: 18px;
+    }
+
+    .mobile-column-actions-dialog .col-actions a,
+    .mobile-column-actions-dialog .col-actions button {
+        padding-left: 34px;
+        padding-right: 18px;
+    }
+
+    .mobile-column-description,
+    .mobile-column-no-actions {
+        padding-left: 18px;
+        padding-right: 18px;
+    }
+
+    dialog.set-column-type-dialog {
+        width: 95vw;
+        max-height: 85vh;
+        border-radius: 0.5rem;
+    }
+
+    .set-column-type-dialog .modal-header,
+    .set-column-type-status,
+    .set-column-type-empty,
+    .set-column-type-error,
+    .set-column-type-options {
+        padding-left: 18px;
+        padding-right: 18px;
+    }
+}
+
 @media only screen and (max-width: 576px) {
 
     .small-screen-only {
@@ -791,6 +1297,43 @@ p.zero-results {
     .filters input.filter-value {
         width: 140px;
     }
+    button.choose-columns-mobile,
+    button.column-actions-mobile {
+        display: inline-flex;
+        align-items: center;
+        justify-content: center;
+        padding: 0.5rem 1rem;
+        margin-bottom: 1em;
+        font-size: 0.9rem;
+        line-height: 1.2;
+        font-family: inherit;
+        background: white;
+        border: 1px solid #ccc;
+        border-radius: 5px;
+        cursor: pointer;
+        vertical-align: top;
+        box-sizing: border-box;
+        min-height: 2.5rem;
+    }
+
+    button.column-actions-mobile {
+        gap: 0.55rem;
+    }
+
+    button.column-actions-mobile svg {
+        display: block;
+        width: 16px;
+        height: 16px;
+        flex-shrink: 0;
+    }
+
+    button.column-actions-mobile span {
+        line-height: 1.2;
+    }
+
+    button.choose-columns-mobile {
+        margin-right: 0.5rem;
+    }
 }
 
 svg.dropdown-menu-icon {
@@ -866,11 +1409,15 @@ svg.dropdown-menu-icon {
     border-bottom: 5px solid #666;
 }
 
-.canned-query-edit-sql {
+.stored-query-edit-sql {
     padding-left: 0.5em;
     position: relative;
     top: 1px;
 }
+.save-query {
+    display: inline-block;
+    margin-left: 0.45em;
+}
 
 .blob-download {
     display: block;

datasette/static/column-chooser.js

@@ -0,0 +1,699 @@
+class ColumnChooser extends HTMLElement {
+  constructor() {
+    super();
+    this.attachShadow({ mode: "open" });
+
+    // State
+    this._items = [];
+    this._checked = new Set();
+    this._savedItems = null;
+    this._savedChecked = null;
+    this._onApply = null;
+
+    // Drag state
+    this._ghost = null;
+    this._dragSrcIdx = null;
+    this._dropTargetIdx = null;
+    this._dropPosition = null;
+    this._ghostOffX = 0;
+    this._ghostOffY = 0;
+    this._autoScrollRAF = null;
+    this._lastPointerY = 0;
+    this._lastPointerX = 0;
+    this._SCROLL_ZONE = 72;
+    this._SCROLL_SPEED = 0.4;
+
+    // Bound handlers
+    this._onMove = this._onMove.bind(this);
+    this._onUp = this._onUp.bind(this);
+
+    this.shadowRoot.innerHTML = `
+      <style>
+        :host {
+          --ink: #0f0f0f;
+          --paper: #f5f3ef;
+          --muted: #6b6b6b;
+          --rule: #e2dfd8;
+          --accent: #1a56db;
+          --accent-light: #e8effd;
+          --card: #ffffff;
+        }
+
+        * { box-sizing: border-box; margin: 0; padding: 0; }
+
+        dialog {
+          border: none;
+          border-radius: var(--modal-border-radius, 0.75rem);
+          padding: 0;
+          margin: auto;
+          width: 100%;
+          max-width: 420px;
+          max-height: min(640px, calc(100vh - 32px));
+          box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
+          animation: slideIn var(--modal-animation-duration, 0.2s) ease-out;
+          overflow: hidden;
+          font-family: system-ui, -apple-system, sans-serif;
+          background: var(--card);
+          -webkit-user-select: none;
+          -webkit-touch-callout: none;
+          -webkit-tap-highlight-color: transparent;
+        }
+
+        dialog[open] {
+          display: flex;
+          flex-direction: column;
+          height: min(640px, calc(100vh - 32px));
+        }
+
+        dialog::backdrop {
+          background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
+          backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+          -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+          animation: fadeIn var(--modal-animation-duration, 0.2s) ease-out;
+        }
+
+        @keyframes slideIn {
+          from {
+            opacity: 0;
+            transform: translateY(-20px) scale(0.95);
+          }
+          to {
+            opacity: 1;
+            transform: translateY(0) scale(1);
+          }
+        }
+
+        @keyframes fadeIn {
+          from { opacity: 0; }
+          to { opacity: 1; }
+        }
+
+        .modal-header {
+          padding: 20px 24px 16px;
+          border-bottom: 1px solid var(--rule);
+          display: flex;
+          align-items: center;
+          justify-content: space-between;
+          flex-shrink: 0;
+        }
+
+        .modal-title {
+          font-size: 1rem;
+          font-weight: 600;
+        }
+
+        .modal-meta {
+          font-family: ui-monospace, monospace;
+          font-size: 0.7rem;
+          color: var(--muted);
+          background: var(--paper);
+          padding: 3px 9px;
+          border-radius: 20px;
+        }
+
+        .list-toolbar {
+          padding: 6px 24px;
+          border-bottom: 1px solid var(--rule);
+          display: flex;
+          gap: 12px;
+          flex-shrink: 0;
+        }
+
+        .list-toolbar button {
+          background: var(--accent-light);
+          border: 1px solid var(--rule);
+          border-radius: 4px;
+          font-family: inherit;
+          font-size: 0.75rem;
+          color: var(--accent);
+          cursor: pointer;
+          padding: 3px 10px;
+          transition: background 0.12s, color 0.12s;
+        }
+        .list-toolbar button:hover { background: var(--accent); color: white; }
+
+        .list-wrap {
+          flex: 1;
+          overflow-y: auto;
+          overflow-x: hidden;
+          position: relative;
+          overscroll-behavior: contain;
+          -webkit-overflow-scrolling: touch;
+        }
+
+        .list-wrap::before,
+        .list-wrap::after {
+          content: '';
+          position: sticky;
+          display: block;
+          left: 0; right: 0;
+          height: 20px;
+          pointer-events: none;
+          z-index: 5;
+          transition: opacity 0.2s;
+        }
+        .list-wrap::before {
+          top: 0;
+          background: linear-gradient(to bottom, rgba(255,255,255,0.9), transparent);
+        }
+        .list-wrap::after {
+          bottom: 0;
+          background: linear-gradient(to top, rgba(255,255,255,0.9), transparent);
+          margin-top: -20px;
+        }
+
+        .scroll-zone {
+          position: absolute;
+          left: 0; right: 0;
+          height: 72px;
+          pointer-events: none;
+          z-index: 10;
+        }
+        .scroll-zone-top { top: 0; }
+        .scroll-zone-bot { bottom: 0; }
+
+        .drag-list {
+          list-style: none;
+          padding: 4px 0;
+        }
+
+        .drag-item {
+          display: flex;
+          align-items: center;
+          background: white;
+          border-bottom: 1px solid var(--rule);
+          user-select: none;
+          -webkit-user-select: none;
+          -webkit-touch-callout: none;
+          position: relative;
+          transition: background 0.08s;
+        }
+
+        .drag-item:last-child { border-bottom: none; }
+
+        .drag-handle {
+          display: flex;
+          align-items: center;
+          justify-content: center;
+          width: 48px;
+          height: 48px;
+          flex-shrink: 0;
+          cursor: grab;
+          color: #c8c4bc;
+          touch-action: none;
+          transition: color 0.15s;
+        }
+
+        .drag-handle:hover { color: var(--accent); }
+        .drag-handle svg { pointer-events: none; display: block; }
+
+        .drag-item-content {
+          display: flex;
+          align-items: center;
+          flex: 1;
+          min-width: 0;
+          cursor: pointer;
+        }
+
+        .drag-item-check {
+          display: flex;
+          align-items: center;
+          width: 32px;
+          height: 48px;
+          flex-shrink: 0;
+        }
+
+        .drag-item-check input[type="checkbox"] {
+          width: 16px;
+          height: 16px;
+          accent-color: var(--accent);
+          cursor: pointer;
+        }
+
+        .drag-item-label {
+          flex: 1;
+          font-size: 0.9rem;
+          line-height: 48px;
+          padding-right: 16px;
+          white-space: nowrap;
+          overflow: hidden;
+          text-overflow: ellipsis;
+          cursor: default;
+        }
+
+        .drag-item.is-dragging {
+          opacity: 0;
+        }
+
+        .drop-indicator {
+          position: absolute;
+          left: 48px;
+          right: 0;
+          height: 2px;
+          background: var(--accent);
+          border-radius: 99px;
+          pointer-events: none;
+          z-index: 20;
+          display: none;
+        }
+        .drop-indicator.top { top: -1px; display: block; }
+        .drop-indicator.bottom { bottom: -1px; display: block; }
+
+        .drag-ghost {
+          position: fixed;
+          pointer-events: none;
+          z-index: 9999;
+          background: white;
+          border-radius: 6px;
+          box-shadow: 0 8px 32px rgba(0,0,0,0.18), 0 2px 8px rgba(0,0,0,0.1);
+          display: flex;
+          align-items: center;
+          border: 1.5px solid var(--accent-light);
+          opacity: 0.97;
+          will-change: transform;
+          font-family: system-ui, -apple-system, sans-serif;
+        }
+
+        .scroll-pulse {
+          position: absolute;
+          left: 50%;
+          transform: translateX(-50%);
+          width: 32px;
+          height: 32px;
+          border-radius: 50%;
+          background: var(--accent);
+          opacity: 0;
+          pointer-events: none;
+          z-index: 10;
+          transition: opacity 0.15s;
+        }
+        .scroll-pulse.top { top: 8px; }
+        .scroll-pulse.bot { bottom: 8px; }
+        .scroll-pulse.active {
+          opacity: 0.18;
+          animation: pulse 0.8s ease-in-out infinite;
+        }
+
+        @keyframes pulse {
+          0%, 100% { transform: translateX(-50%) scale(1); opacity: 0.18; }
+          50% { transform: translateX(-50%) scale(1.5); opacity: 0.07; }
+        }
+
+        .modal-footer {
+          padding: 14px 20px;
+          border-top: 1px solid var(--rule);
+          display: flex;
+          align-items: center;
+          gap: 10px;
+          flex-shrink: 0;
+          background: var(--paper);
+        }
+
+        .footer-info {
+          flex: 1;
+          font-family: ui-monospace, monospace;
+          font-size: 0.68rem;
+          color: var(--muted);
+        }
+
+        .btn {
+          border: none;
+          border-radius: 5px;
+          padding: 9px 20px;
+          font-size: 0.85rem;
+          font-weight: 500;
+          cursor: pointer;
+          touch-action: manipulation;
+          font-family: inherit;
+          transition: background 0.12s;
+        }
+
+        .btn-primary {
+          background: var(--accent);
+          color: white;
+        }
+        .btn-primary:hover { background: #1448c0; }
+
+        .btn-ghost {
+          background: transparent;
+          color: var(--muted);
+          border: 1px solid var(--rule);
+        }
+        .btn-ghost:hover { background: var(--rule); color: var(--ink); }
+
+        .list-wrap::-webkit-scrollbar { width: 5px; }
+        .list-wrap::-webkit-scrollbar-track { background: transparent; }
+        .list-wrap::-webkit-scrollbar-thumb { background: var(--rule); border-radius: 99px; }
+
+        input, textarea { -webkit-user-select: auto; user-select: auto; }
+      </style>
+
+      <dialog aria-labelledby="modalTitle">
+          <div class="modal-header">
+            <span class="modal-title" id="modalTitle">Choose columns</span>
+            <span class="modal-meta" id="selectedCount"></span>
+          </div>
+          <div class="list-toolbar">
+            <button id="selectAllBtn">Select all</button>
+            <button id="deselectAllBtn">Deselect all</button>
+          </div>
+          <div class="list-wrap" id="listWrap">
+            <div class="scroll-pulse top" id="pulseTop"></div>
+            <div class="scroll-pulse bot" id="pulseBot"></div>
+            <ul class="drag-list" id="dragList"></ul>
+          </div>
+          <div class="modal-footer">
+            <span class="footer-info" id="footerInfo"></span>
+            <button class="btn btn-ghost" id="cancelBtn">Cancel</button>
+            <button class="btn btn-primary" id="applyBtn">Apply</button>
+          </div>
+      </dialog>
+    `;
+
+    // DOM refs
+    this._dialog = this.shadowRoot.querySelector("dialog");
+    this._listWrap = this.shadowRoot.getElementById("listWrap");
+    this._dragList = this.shadowRoot.getElementById("dragList");
+    this._pulseTop = this.shadowRoot.getElementById("pulseTop");
+    this._pulseBot = this.shadowRoot.getElementById("pulseBot");
+    this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn");
+    this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn");
+    this._cancelBtn = this.shadowRoot.getElementById("cancelBtn");
+    this._applyBtn = this.shadowRoot.getElementById("applyBtn");
+    this._countEl = this.shadowRoot.getElementById("selectedCount");
+    this._footerEl = this.shadowRoot.getElementById("footerInfo");
+
+    // Event listeners
+    this._selectAllBtn.addEventListener("click", () => this._selectAll());
+    this._deselectAllBtn.addEventListener("click", () => this._deselectAll());
+    this._cancelBtn.addEventListener("click", () => this._close());
+    this._applyBtn.addEventListener("click", () => this._apply());
+    this._dialog.addEventListener("click", (e) => {
+      if (e.target === this._dialog) this._close();
+    });
+    this._dialog.addEventListener("cancel", (e) => {
+      e.preventDefault();
+      this._close();
+    });
+  }
+
+  /**
+   * Open the column chooser dialog.
+   * @param {Object} opts
+   * @param {string[]} opts.columns - All available column names, in display order.
+   * @param {string[]} opts.selected - Column names that should be pre-checked.
+   * @param {function(string[]): void} opts.onApply - Called with the selected columns in order when Apply is clicked.
+   */
+  open({ columns, selected = [], onApply }) {
+    this._items = [...columns];
+    this._checked = new Set(selected);
+    this._onApply = onApply || null;
+
+    // Save state for cancel/restore
+    this._savedItems = [...this._items];
+    this._savedChecked = new Set(this._checked);
+
+    this._render();
+    this._dialog.showModal();
+  }
+
+  // ── Internal methods ──
+
+  _close() {
+    this._items = this._savedItems ? [...this._savedItems] : this._items;
+    this._checked = this._savedChecked
+      ? new Set(this._savedChecked)
+      : this._checked;
+    this._dialog.close();
+  }
+
+  _selectAll() {
+    this._items.forEach((col) => this._checked.add(col));
+    this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => {
+      cb.checked = true;
+    });
+    this._updateCounts();
+  }
+
+  _deselectAll() {
+    this._checked.clear();
+    this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => {
+      cb.checked = false;
+    });
+    this._updateCounts();
+  }
+
+  _apply() {
+    const selected = this._items.filter((col) => this._checked.has(col));
+    this._dialog.close();
+    if (this._onApply) {
+      this._onApply(selected);
+    }
+  }
+
+  _render() {
+    this._dragList.innerHTML = "";
+    this._items.forEach((col, i) => {
+      const li = document.createElement("li");
+      li.className = "drag-item";
+      li.dataset.idx = i;
+      li.innerHTML = `
+        <span class="drag-handle" aria-label="Drag to reorder">
+          <svg width="12" height="18" viewBox="0 0 12 18" fill="currentColor">
+            <circle cx="3.5" cy="3.5" r="1.8"/>
+            <circle cx="8.5" cy="3.5" r="1.8"/>
+            <circle cx="3.5" cy="9" r="1.8"/>
+            <circle cx="8.5" cy="9" r="1.8"/>
+            <circle cx="3.5" cy="14.5" r="1.8"/>
+            <circle cx="8.5" cy="14.5" r="1.8"/>
+          </svg>
+        </span>
+        <label class="drag-item-content">
+          <span class="drag-item-check">
+            <input type="checkbox" ${this._checked.has(col) ? "checked" : ""}>
+          </span>
+          <span class="drag-item-label">${col}</span>
+        </label>
+        <div class="drop-indicator"></div>
+      `;
+
+      li.querySelector("input").addEventListener("change", (e) => {
+        e.target.checked ? this._checked.add(col) : this._checked.delete(col);
+        this._updateCounts();
+      });
+
+      li.querySelector(".drag-handle").addEventListener("pointerdown", (e) =>
+        this._startDrag(e, i),
+      );
+      this._dragList.appendChild(li);
+    });
+
+    this._updateCounts();
+  }
+
+  _updateCounts() {
+    const n = this._checked.size;
+    this._countEl.textContent = `${n} of ${this._items.length} selected`;
+    this._footerEl.textContent = `${this._items.length} columns`;
+  }
+
+  // ── Drag engine ──
+
+  _startDrag(e, idx) {
+    e.preventDefault();
+    this._dragSrcIdx = idx;
+
+    const srcEl = this._dragList.children[idx];
+    const rect = srcEl.getBoundingClientRect();
+
+    this._ghostOffX = e.clientX - rect.left;
+    this._ghostOffY = e.clientY - rect.top;
+
+    // Build ghost inside shadow DOM
+    this._ghost = document.createElement("div");
+    this._ghost.className = "drag-ghost";
+    this._ghost.style.width = rect.width + "px";
+    this._ghost.style.height = rect.height + "px";
+    this._ghost.innerHTML = srcEl.innerHTML;
+    this._ghost.querySelector(".drop-indicator")?.remove();
+    const h = this._ghost.querySelector(".drag-handle");
+    if (h) h.style.color = "var(--accent)";
+    this.shadowRoot.appendChild(this._ghost);
+
+    srcEl.classList.add("is-dragging");
+    this._positionGhost(e.clientX, e.clientY);
+
+    document.addEventListener("pointermove", this._onMove);
+    document.addEventListener("pointerup", this._onUp);
+    document.addEventListener("pointercancel", this._onUp);
+  }
+
+  _positionGhost(cx, cy) {
+    this._ghost.style.left = cx - this._ghostOffX + "px";
+    this._ghost.style.top = cy - this._ghostOffY + "px";
+  }
+
+  _onMove(e) {
+    this._lastPointerX = e.clientX;
+    this._lastPointerY = e.clientY;
+    this._positionGhost(e.clientX, e.clientY);
+    this._updateDropTarget(e.clientY);
+    this._updateAutoScroll(e.clientY);
+  }
+
+  _onUp() {
+    document.removeEventListener("pointermove", this._onMove);
+    document.removeEventListener("pointerup", this._onUp);
+    document.removeEventListener("pointercancel", this._onUp);
+
+    this._stopAutoScroll();
+
+    const noMove =
+      this._dropTargetIdx === null || this._dropTargetIdx === this._dragSrcIdx;
+    this._clearDropIndicators();
+
+    let dest = null;
+    if (!noMove) {
+      const moved = this._items.splice(this._dragSrcIdx, 1)[0];
+      dest = this._dropTargetIdx;
+      if (this._dropPosition === "after") dest++;
+      if (dest > this._dragSrcIdx) dest--;
+      this._items.splice(dest, 0, moved);
+    }
+
+    this._dragSrcIdx = null;
+    this._dropTargetIdx = null;
+    this._dropPosition = null;
+
+    const g = this._ghost;
+    this._ghost = null;
+
+    if (noMove) {
+      if (g) g.remove();
+      this._render();
+      return;
+    }
+
+    this._render();
+
+    if (g && dest !== null) {
+      const landedEl = this._dragList.children[dest];
+      if (landedEl) {
+        landedEl.style.opacity = "0";
+        const r = landedEl.getBoundingClientRect();
+        g.getBoundingClientRect();
+        g.style.transition =
+          "left 0.15s cubic-bezier(0.22, 1, 0.36, 1), top 0.15s cubic-bezier(0.22, 1, 0.36, 1), box-shadow 0.15s, opacity 0.1s 0.1s";
+        g.style.left = r.left + "px";
+        g.style.top = r.top + "px";
+        g.style.boxShadow = "0 1px 4px rgba(0,0,0,0.08)";
+        g.style.opacity = "0";
+        setTimeout(() => {
+          g.remove();
+          if (landedEl) landedEl.style.opacity = "";
+        }, 160);
+      } else {
+        g.remove();
+      }
+    } else if (g) {
+      g.remove();
+    }
+  }
+
+  _updateDropTarget(clientY) {
+    this._clearDropIndicators();
+    const listItems = [
+      ...this._dragList.querySelectorAll(".drag-item:not(.is-dragging)"),
+    ];
+    if (!listItems.length) return;
+
+    let best = null,
+      bestDist = Infinity;
+    listItems.forEach((li) => {
+      const r = li.getBoundingClientRect();
+      const mid = r.top + r.height / 2;
+      const dist = Math.abs(clientY - mid);
+      if (dist < bestDist) {
+        bestDist = dist;
+        best = li;
+      }
+    });
+
+    if (!best) return;
+    const r = best.getBoundingClientRect();
+    const mid = r.top + r.height / 2;
+    const above = clientY < mid;
+    const indic = best.querySelector(".drop-indicator");
+
+    this._dropTargetIdx = parseInt(best.dataset.idx);
+    this._dropPosition = above ? "before" : "after";
+
+    if (indic) {
+      indic.className = "drop-indicator " + (above ? "top" : "bottom");
+    }
+  }
+
+  _clearDropIndicators() {
+    this._dragList.querySelectorAll(".drop-indicator").forEach((el) => {
+      el.className = "drop-indicator";
+    });
+  }
+
+  _updateAutoScroll(clientY) {
+    const rect = this._listWrap.getBoundingClientRect();
+    const relY = clientY - rect.top;
+    const distTop = relY;
+    const distBot = rect.height - relY;
+
+    const inTop = distTop < this._SCROLL_ZONE && distTop >= 0;
+    const inBot = distBot < this._SCROLL_ZONE && distBot >= 0;
+
+    this._pulseTop.classList.toggle("active", inTop);
+    this._pulseBot.classList.toggle("active", inBot);
+
+    if ((inTop || inBot) && !this._autoScrollRAF) {
+      let lastTime = null;
+      const loop = (ts) => {
+        if (!this._ghost) {
+          this._stopAutoScroll();
+          return;
+        }
+        if (lastTime !== null) {
+          const dt = ts - lastTime;
+          const rect2 = this._listWrap.getBoundingClientRect();
+          const relY2 = this._lastPointerY - rect2.top;
+          const dTop = relY2;
+          const dBot = rect2.height - relY2;
+
+          if (dTop < this._SCROLL_ZONE && dTop >= 0) {
+            const factor = 1 - dTop / this._SCROLL_ZONE;
+            this._listWrap.scrollTop -= this._SCROLL_SPEED * dt * factor * 2.5;
+          } else if (dBot < this._SCROLL_ZONE && dBot >= 0) {
+            const factor = 1 - dBot / this._SCROLL_ZONE;
+            this._listWrap.scrollTop += this._SCROLL_SPEED * dt * factor * 2.5;
+          } else {
+            this._stopAutoScroll();
+            return;
+          }
+          this._updateDropTarget(this._lastPointerY);
+        }
+        lastTime = ts;
+        this._autoScrollRAF = requestAnimationFrame(loop);
+      };
+      this._autoScrollRAF = requestAnimationFrame(loop);
+    }
+
+    if (!inTop && !inBot) this._stopAutoScroll();
+  }
+
+  _stopAutoScroll() {
+    if (this._autoScrollRAF) {
+      cancelAnimationFrame(this._autoScrollRAF);
+      this._autoScrollRAF = null;
+    }
+    this._pulseTop.classList.remove("active");
+    this._pulseBot.classList.remove("active");
+  }
+}
+
+customElements.define("column-chooser", ColumnChooser);

datasette/static/datasette-manager.js

@@ -82,6 +82,19 @@ const datasetteManager = {
     return columnActions;
   },
 
+  makeJumpSections: (context) => {
+    let jumpSections = [];
+
+    datasetteManager.plugins.forEach((plugin) => {
+      if (plugin.makeJumpSections) {
+        const sections = plugin.makeJumpSections(context) || [];
+        jumpSections.push(...sections);
+      }
+    });
+
+    return jumpSections;
+  },
+
   /**
    * In MVP, each plugin can only have 1 instance.
    * In future, panels could be repeated. We omit that for now since so many plugins depend on
@@ -192,7 +205,6 @@ const initializeDatasette = () => {
   // DATASETTE_EVENTS.INIT event to avoid the habit of reading from the window.
 
   window.__DATASETTE__ = datasetteManager;
-  console.debug("Datasette Manager Created!");
 
   const initDatasetteEvent = new CustomEvent(DATASETTE_EVENTS.INIT, {
     detail: datasetteManager,

datasette/static/mobile-column-actions.js

@@ -0,0 +1,318 @@
+var MOBILE_COLUMN_BREAKPOINT = 576;
+var MOBILE_COLUMN_DIALOG_ID = "mobile-column-actions-dialog";
+var MOBILE_COLUMN_DIALOG_TITLE_ID = "mobile-column-actions-title";
+
+function mobileColumnHeaders(manager) {
+  return Array.from(
+    document.querySelectorAll(manager.selectors.tableHeaders),
+  ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1");
+}
+
+function mobileColumnMetaText(th) {
+  var parts = [];
+  if (th.dataset.columnType) {
+    parts.push(th.dataset.columnType);
+  }
+  if (th.dataset.isPk === "1") {
+    parts.push("pk");
+  }
+  if (th.dataset.columnNotNull === "1") {
+    parts.push("not null");
+  }
+  return parts.join(", ");
+}
+
+function createMobileColumnActionNode(itemConfig, closeDialog) {
+  var actionNode;
+  if (itemConfig.href) {
+    actionNode = document.createElement("a");
+    actionNode.href = itemConfig.href;
+  } else {
+    actionNode = document.createElement("button");
+    actionNode.type = "button";
+  }
+  actionNode.textContent = itemConfig.label;
+
+  if (itemConfig.onClick) {
+    actionNode.addEventListener("click", function (ev) {
+      try {
+        itemConfig.onClick.call(actionNode, ev);
+      } finally {
+        closeDialog({ restoreFocus: false });
+      }
+    });
+  }
+
+  return actionNode;
+}
+
+function initMobileColumnActions(manager) {
+  var triggerButton = document.querySelector(".column-actions-mobile");
+  if (!triggerButton) {
+    return;
+  }
+
+  if (
+    !window.URLSearchParams ||
+    !window.HTMLDialogElement ||
+    !manager.columnActions
+  ) {
+    triggerButton.style.display = "none";
+    return;
+  }
+
+  if (!mobileColumnHeaders(manager).length) {
+    triggerButton.style.display = "none";
+    return;
+  }
+
+  var dialog = document.createElement("dialog");
+  dialog.className = "mobile-column-actions-dialog";
+  dialog.id = MOBILE_COLUMN_DIALOG_ID;
+  dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID);
+  dialog.innerHTML = `
+    <div class="modal-header">
+      <span class="modal-title" id="${MOBILE_COLUMN_DIALOG_TITLE_ID}">Column actions</span>
+      <span class="modal-meta"></span>
+    </div>
+    <div class="list-wrap mobile-column-list"></div>
+    <div class="modal-footer">
+      <span class="footer-info">Tap a column to reveal actions.</span>
+      <button type="button" class="btn btn-ghost mobile-column-actions-done">Done</button>
+    </div>
+  `;
+  document.body.appendChild(dialog);
+
+  triggerButton.setAttribute("aria-haspopup", "dialog");
+  triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID);
+  triggerButton.setAttribute("aria-expanded", "false");
+
+  var countEl = dialog.querySelector(".modal-meta");
+  var listWrap = dialog.querySelector(".mobile-column-list");
+  var doneButton = dialog.querySelector(".mobile-column-actions-done");
+  var expandedSectionId = null;
+  var shouldRestoreFocus = true;
+
+  function updateExpandedSection() {
+    Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => {
+      var controlsId = button.getAttribute("aria-controls");
+      var actionList = dialog.querySelector("#" + controlsId);
+      var isExpanded = controlsId === expandedSectionId;
+      button.setAttribute("aria-expanded", isExpanded ? "true" : "false");
+      actionList.hidden = !isExpanded;
+      actionList.classList.toggle("expanded", isExpanded);
+    });
+  }
+
+  function scrollExpandedSectionIntoView(section) {
+    var sectionTop = section.offsetTop;
+    var sectionBottom = sectionTop + section.offsetHeight;
+    var visibleTop = listWrap.scrollTop;
+    var visibleBottom = visibleTop + listWrap.clientHeight;
+    var sectionHeight = section.offsetHeight;
+
+    if (sectionTop < visibleTop) {
+      listWrap.scrollTop = sectionTop;
+      return;
+    }
+
+    if (sectionBottom <= visibleBottom) {
+      return;
+    }
+
+    if (sectionHeight <= listWrap.clientHeight) {
+      listWrap.scrollTop = sectionBottom - listWrap.clientHeight;
+    } else {
+      listWrap.scrollTop = sectionTop;
+    }
+  }
+
+  function closeDialog(options) {
+    options = options || {};
+    shouldRestoreFocus = options.restoreFocus !== false;
+    if (dialog.open) {
+      dialog.close();
+    } else {
+      triggerButton.setAttribute("aria-expanded", "false");
+      if (shouldRestoreFocus) {
+        triggerButton.focus();
+      }
+    }
+  }
+
+  function renderDialog() {
+    var headers = mobileColumnHeaders(manager);
+    if (!headers.length) {
+      closeDialog({ restoreFocus: false });
+      triggerButton.style.display = "none";
+      return false;
+    }
+
+    if (
+      !headers.some(
+        (_th, index) => `mobile-column-actions-${index}` === expandedSectionId,
+      )
+    ) {
+      expandedSectionId = null;
+    }
+
+    countEl.textContent = `${headers.length} column${
+      headers.length === 1 ? "" : "s"
+    }`;
+    listWrap.innerHTML = "";
+
+    if (manager.columnActions.shouldShowShowAllColumns()) {
+      var topActions = document.createElement("div");
+      topActions.className = "mobile-column-top-actions";
+
+      var showAllColumns = document.createElement("a");
+      showAllColumns.className = "btn btn-ghost mobile-column-top-action";
+      showAllColumns.href = manager.columnActions.showAllColumnsUrl();
+      showAllColumns.textContent = "Show all columns";
+
+      topActions.appendChild(showAllColumns);
+      listWrap.appendChild(topActions);
+    }
+
+    headers.forEach((th, index) => {
+      var sectionId = `mobile-column-actions-${index}`;
+      var actionState = manager.columnActions.buildColumnActionState(th, {
+        includeChooseColumns: false,
+        includeShowAllColumns: false,
+      });
+      var section = document.createElement("section");
+      section.className = "mobile-column-section";
+
+      var headerButton = document.createElement("button");
+      headerButton.type = "button";
+      headerButton.className = "col-header";
+      headerButton.setAttribute("aria-controls", sectionId);
+      headerButton.setAttribute("aria-expanded", "false");
+
+      var headerText = document.createElement("span");
+      headerText.className = "mobile-column-header-text";
+
+      var name = document.createElement("span");
+      name.className = "mobile-column-name";
+      name.textContent = th.dataset.column;
+      headerText.appendChild(name);
+
+      var metaText = mobileColumnMetaText(th);
+      if (metaText) {
+        var meta = document.createElement("span");
+        meta.className = "mobile-column-meta";
+        meta.textContent = metaText;
+        headerText.appendChild(meta);
+      }
+
+      var chevron = document.createElement("span");
+      chevron.className = "mobile-column-chevron";
+      chevron.setAttribute("aria-hidden", "true");
+      chevron.textContent = "▾";
+
+      headerButton.appendChild(headerText);
+      headerButton.appendChild(chevron);
+      headerButton.addEventListener("click", function () {
+        expandedSectionId = expandedSectionId === sectionId ? null : sectionId;
+        updateExpandedSection();
+        if (expandedSectionId === sectionId) {
+          scrollExpandedSectionIntoView(section);
+        }
+      });
+
+      var actionContainer = document.createElement("div");
+      actionContainer.id = sectionId;
+      actionContainer.className = "col-actions";
+      actionContainer.hidden = true;
+
+      if (actionState.columnDescription) {
+        var description = document.createElement("p");
+        description.className = "mobile-column-description";
+        description.textContent = actionState.columnDescription;
+        actionContainer.appendChild(description);
+      }
+
+      if (actionState.actionItems.length) {
+        var actionList = document.createElement("ul");
+        actionState.actionItems.forEach((itemConfig) => {
+          var actionItem = document.createElement("li");
+          actionItem.appendChild(
+            createMobileColumnActionNode(itemConfig, closeDialog),
+          );
+          actionList.appendChild(actionItem);
+        });
+        actionContainer.appendChild(actionList);
+      } else {
+        var noActions = document.createElement("p");
+        noActions.className = "mobile-column-no-actions";
+        noActions.textContent = "No actions available";
+        actionContainer.appendChild(noActions);
+      }
+
+      section.appendChild(headerButton);
+      section.appendChild(actionContainer);
+      listWrap.appendChild(section);
+    });
+
+    updateExpandedSection();
+    return true;
+  }
+
+  function openDialog() {
+    if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT) {
+      return;
+    }
+    if (!renderDialog()) {
+      return;
+    }
+    if (!dialog.open) {
+      dialog.showModal();
+    }
+    triggerButton.setAttribute("aria-expanded", "true");
+    var focusTarget =
+      dialog.querySelector(".mobile-column-top-action") ||
+      dialog.querySelector(".col-header") ||
+      doneButton;
+    focusTarget.focus();
+  }
+
+  triggerButton.addEventListener("click", function () {
+    if (dialog.open) {
+      closeDialog();
+    } else {
+      openDialog();
+    }
+  });
+
+  doneButton.addEventListener("click", function () {
+    closeDialog();
+  });
+
+  dialog.addEventListener("click", function (ev) {
+    if (ev.target === dialog) {
+      closeDialog();
+    }
+  });
+
+  dialog.addEventListener("cancel", function (ev) {
+    ev.preventDefault();
+    closeDialog();
+  });
+
+  dialog.addEventListener("close", function () {
+    triggerButton.setAttribute("aria-expanded", "false");
+    if (shouldRestoreFocus) {
+      triggerButton.focus();
+    }
+  });
+
+  window.addEventListener("resize", function () {
+    if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT && dialog.open) {
+      closeDialog({ restoreFocus: false });
+    }
+  });
+}
+
+document.addEventListener("datasette_init", function (evt) {
+  initMobileColumnActions(evt.detail);
+});

datasette/static/navigation-search.js

@@ -1,10 +1,22 @@
+let navigationSearchInstanceCounter = 0;
+
 class NavigationSearch extends HTMLElement {
   constructor() {
     super();
+    this.instanceId = ++navigationSearchInstanceCounter;
+    this.inputId = `navigation-search-input-${this.instanceId}`;
+    this.instructionsId = `navigation-search-instructions-${this.instanceId}`;
+    this.listboxId = `navigation-search-results-${this.instanceId}`;
+    this.recentHeadingId = `navigation-search-recent-${this.instanceId}`;
+    this.statusId = `navigation-search-status-${this.instanceId}`;
+    this.titleId = `navigation-search-title-${this.instanceId}`;
     this.attachShadow({ mode: "open" });
     this.selectedIndex = -1;
     this.matches = [];
+    this.renderedMatches = [];
     this.debounceTimer = null;
+    this.restoreFocusTarget = null;
+    this.shouldRestoreFocus = true;
 
     this.render();
     this.setupEventListeners();
@@ -19,19 +31,20 @@ class NavigationSearch extends HTMLElement {
 
                 dialog {
                     border: none;
-                    border-radius: 0.75rem;
+                    border-radius: var(--modal-border-radius, 0.75rem);
                     padding: 0;
                     max-width: 90vw;
                     width: 600px;
                     max-height: 80vh;
-                    box-shadow: 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04);
-                    animation: slideIn 0.2s ease-out;
+                    box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
+                    animation: slideIn var(--modal-animation-duration, 0.2s) ease-out;
                 }
 
                 dialog::backdrop {
-                    background: rgba(0, 0, 0, 0.5);
-                    backdrop-filter: blur(4px);
-                    animation: fadeIn 0.2s ease-out;
+                    background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
+                    backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+                    -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+                    animation: fadeIn var(--modal-animation-duration, 0.2s) ease-out;
                 }
 
                 @keyframes slideIn {
@@ -53,16 +66,20 @@ class NavigationSearch extends HTMLElement {
                 .search-container {
                     display: flex;
                     flex-direction: column;
-                    height: 100%;
                 }
 
                 .search-input-wrapper {
                     padding: 1.25rem;
                     border-bottom: 1px solid #e5e7eb;
+                    display: flex;
+                    gap: 0.5rem;
+                    align-items: center;
                 }
 
                 .search-input {
                     width: 100%;
+                    flex: 1;
+                    min-width: 0;
                     padding: 0.75rem 1rem;
                     font-size: 1rem;
                     border: 2px solid #e5e7eb;
@@ -76,12 +93,36 @@ class NavigationSearch extends HTMLElement {
                     border-color: #2563eb;
                 }
 
+                .close-search {
+                    background: transparent;
+                    border: 1px solid transparent;
+                    border-radius: 0.375rem;
+                    color: #4b5563;
+                    cursor: pointer;
+                    flex: 0 0 auto;
+                    font: inherit;
+                    font-size: 1.5rem;
+                    height: 2.75rem;
+                    line-height: 1;
+                    width: 2.75rem;
+                }
+
+                .close-search:hover,
+                .close-search:focus {
+                    background-color: #f3f4f6;
+                    border-color: #d1d5db;
+                }
+
                 .results-container {
                     overflow-y: auto;
                     height: calc(80vh - 180px);
                     padding: 0.5rem;
                 }
 
+                .results-list:empty {
+                    display: none;
+                }
+
                 .result-item {
                     padding: 0.875rem 1rem;
                     cursor: pointer;
@@ -100,16 +141,81 @@ class NavigationSearch extends HTMLElement {
                     background-color: #dbeafe;
                 }
 
+                .result-item > div {
+                    flex: 1;
+                    min-width: 0;
+                }
+
+                .jump-start-content {
+                    border-bottom: 1px solid #e5e7eb;
+                    margin-bottom: 0.5rem;
+                    padding: 0.5rem 0.5rem 1rem;
+                }
+
+                .jump-start-content:empty {
+                    display: none;
+                }
+
                 .result-name {
                     font-weight: 500;
                     color: #111827;
                 }
 
+                .result-label {
+                    font-size: 0.875rem;
+                    color: #4b5563;
+                }
+
+                .result-type {
+                    color: #4b5563;
+                    font-size: 0.75rem;
+                    font-weight: 600;
+                    text-transform: uppercase;
+                }
+
                 .result-url {
                     font-size: 0.875rem;
                     color: #6b7280;
                 }
 
+                .result-description {
+                    color: #374151;
+                    display: -webkit-box;
+                    font-size: 0.8125rem;
+                    line-height: 1.35;
+                    margin-top: 0.35rem;
+                    overflow: hidden;
+                    -webkit-box-orient: vertical;
+                    -webkit-line-clamp: 2;
+                }
+
+                .results-heading {
+                    color: #4b5563;
+                    font-size: 0.75rem;
+                    font-weight: 600;
+                    letter-spacing: 0;
+                    padding: 0.5rem 1rem 0.25rem;
+                    text-transform: uppercase;
+                }
+
+                .recent-actions {
+                    padding: 0.25rem 1rem 0.75rem;
+                }
+
+                .clear-recent {
+                    background: transparent;
+                    border: 0;
+                    color: #2563eb;
+                    cursor: pointer;
+                    font: inherit;
+                    font-size: 0.875rem;
+                    padding: 0;
+                }
+
+                .clear-recent:hover {
+                    text-decoration: underline;
+                }
+
                 .no-results {
                     padding: 2rem;
                     text-align: center;
@@ -135,6 +241,18 @@ class NavigationSearch extends HTMLElement {
                     font-family: monospace;
                 }
 
+                .visually-hidden {
+                    border: 0;
+                    clip: rect(0 0 0 0);
+                    height: 1px;
+                    margin: -1px;
+                    overflow: hidden;
+                    padding: 0;
+                    position: absolute;
+                    white-space: nowrap;
+                    width: 1px;
+                }
+
                 /* Mobile optimizations */
                 @media (max-width: 640px) {
                     dialog {
@@ -162,19 +280,29 @@ class NavigationSearch extends HTMLElement {
                 }
             </style>
 
-            <dialog>
+            <dialog aria-modal="true" aria-labelledby="${this.titleId}">
                 <div class="search-container">
+                    <h2 id="${this.titleId}" class="visually-hidden">Jump to</h2>
+                    <p id="${this.instructionsId}" class="visually-hidden">Type to search. Use up and down arrow keys to move through results, Enter to select a result, and Escape to close this menu.</p>
+                    <div id="${this.statusId}" class="visually-hidden" aria-live="polite" aria-atomic="true"></div>
                     <div class="search-input-wrapper">
                         <input 
+                            id="${this.inputId}"
                             type="text" 
                             class="search-input" 
-                            placeholder="Search..."
-                            aria-label="Search navigation"
+                            placeholder="Jump to..."
+                            aria-label="Jump to"
+                            aria-describedby="${this.instructionsId}"
+                            role="combobox"
+                            aria-autocomplete="list"
+                            aria-controls="${this.listboxId}"
+                            aria-expanded="false"
                             autocomplete="off"
                             spellcheck="false"
                         >
+                        <button type="button" class="close-search" aria-label="Close jump menu">&times;</button>
                     </div>
-                    <div class="results-container" role="listbox"></div>
+                    <div class="results-container"></div>
                     <div class="hint-text">
                         <span><kbd>↑</kbd> <kbd>↓</kbd> Navigate</span>
                         <span><kbd>Enter</kbd> Select</span>
@@ -188,6 +316,7 @@ class NavigationSearch extends HTMLElement {
   setupEventListeners() {
     const dialog = this.shadowRoot.querySelector("dialog");
     const input = this.shadowRoot.querySelector(".search-input");
+    const closeButton = this.shadowRoot.querySelector(".close-search");
     const resultsContainer =
       this.shadowRoot.querySelector(".results-container");
 
@@ -199,6 +328,17 @@ class NavigationSearch extends HTMLElement {
       }
     });
 
+    document.addEventListener("click", (e) => {
+      const trigger = e.target.closest("[data-navigation-search-open]");
+      if (trigger) {
+        e.preventDefault();
+        const details = trigger.closest("details");
+        const restoreTarget = details?.querySelector("summary") || trigger;
+        details?.removeAttribute("open");
+        this.openMenu(restoreTarget);
+      }
+    });
+
     // Input event
     input.addEventListener("input", (e) => {
       this.handleSearch(e.target.value);
@@ -220,8 +360,19 @@ class NavigationSearch extends HTMLElement {
       }
     });
 
+    closeButton.addEventListener("click", () => {
+      this.closeMenu();
+    });
+
     // Click on result item
     resultsContainer.addEventListener("click", (e) => {
+      const clearRecent = e.target.closest("[data-clear-recent-items]");
+      if (clearRecent) {
+        e.preventDefault();
+        this.clearRecentItems();
+        return;
+      }
+
       const item = e.target.closest(".result-item");
       if (item) {
         const index = parseInt(item.dataset.index);
@@ -236,6 +387,15 @@ class NavigationSearch extends HTMLElement {
       }
     });
 
+    dialog.addEventListener("cancel", (e) => {
+      e.preventDefault();
+      this.closeMenu();
+    });
+
+    dialog.addEventListener("close", () => {
+      this.onMenuClosed();
+    });
+
     // Initial load
     this.loadInitialData();
   }
@@ -250,6 +410,106 @@ class NavigationSearch extends HTMLElement {
     );
   }
 
+  setElementAttribute(element, name, value) {
+    if (!element) {
+      return;
+    }
+    if (typeof element.setAttribute === "function") {
+      element.setAttribute(name, value);
+    } else {
+      element[name] = String(value);
+    }
+  }
+
+  removeElementAttribute(element, name) {
+    if (!element) {
+      return;
+    }
+    if (typeof element.removeAttribute === "function") {
+      element.removeAttribute(name);
+    } else {
+      delete element[name];
+    }
+  }
+
+  focusRestoreTarget(trigger) {
+    if (trigger && typeof trigger.focus === "function") {
+      return trigger;
+    }
+    if (
+      document.activeElement &&
+      typeof document.activeElement.focus === "function"
+    ) {
+      return document.activeElement;
+    }
+    return null;
+  }
+
+  setNavigationTriggersExpanded(expanded) {
+    if (typeof document.querySelectorAll !== "function") {
+      return;
+    }
+    document
+      .querySelectorAll("[data-navigation-search-open]")
+      .forEach((trigger) => {
+        this.setElementAttribute(
+          trigger,
+          "aria-expanded",
+          expanded ? "true" : "false",
+        );
+      });
+  }
+
+  resultOptionId(index) {
+    return `${this.listboxId}-option-${index}`;
+  }
+
+  updateComboboxState() {
+    const dialog = this.shadowRoot.querySelector("dialog");
+    const input = this.shadowRoot.querySelector(".search-input");
+    const matches = this.renderedMatches || [];
+    this.setElementAttribute(
+      input,
+      "aria-expanded",
+      dialog && dialog.open && matches.length > 0 ? "true" : "false",
+    );
+
+    if (
+      dialog &&
+      dialog.open &&
+      this.selectedIndex >= 0 &&
+      this.selectedIndex < matches.length
+    ) {
+      this.setElementAttribute(
+        input,
+        "aria-activedescendant",
+        this.resultOptionId(this.selectedIndex),
+      );
+    } else {
+      this.removeElementAttribute(input, "aria-activedescendant");
+    }
+  }
+
+  setStatus(message) {
+    const status = this.shadowRoot.querySelector(`#${this.statusId}`);
+    if (status) {
+      status.textContent = message || "";
+    }
+  }
+
+  resultsStatus(count, truncated) {
+    if (truncated) {
+      return "More than 100 results. Keep typing to narrow the list.";
+    }
+    if (count === 0) {
+      return "No results found.";
+    }
+    if (count === 1) {
+      return "1 result.";
+    }
+    return `${count} results.`;
+  }
+
   loadInitialData() {
     const itemsAttr = this.getAttribute("items");
     if (itemsAttr) {
@@ -266,6 +526,11 @@ class NavigationSearch extends HTMLElement {
 
   handleSearch(query) {
     clearTimeout(this.debounceTimer);
+    if (query.trim()) {
+      this.setStatus("Searching...");
+    } else {
+      this.setStatus("");
+    }
 
     this.debounceTimer = setTimeout(() => {
       const url = this.getAttribute("url");
@@ -288,65 +553,262 @@ class NavigationSearch extends HTMLElement {
       this.matches = data.matches || [];
       this.selectedIndex = this.matches.length > 0 ? 0 : -1;
       this.renderResults();
+      if (query.trim()) {
+        this.setStatus(this.resultsStatus(this.matches.length, data.truncated));
+      } else {
+        this.setStatus("");
+      }
     } catch (e) {
       console.error("Failed to fetch search results:", e);
       this.matches = [];
       this.renderResults();
+      this.setStatus("Search failed.");
     }
   }
 
   filterLocalItems(query) {
     if (!query.trim()) {
-      this.matches = [];
+      this.matches = this.allItems || [];
     } else {
       const lowerQuery = query.toLowerCase();
       this.matches = (this.allItems || []).filter(
         (item) =>
           item.name.toLowerCase().includes(lowerQuery) ||
+          (item.display_name || "").toLowerCase().includes(lowerQuery) ||
           item.url.toLowerCase().includes(lowerQuery),
       );
     }
     this.selectedIndex = this.matches.length > 0 ? 0 : -1;
     this.renderResults();
+    if (query.trim()) {
+      this.setStatus(this.resultsStatus(this.matches.length, false));
+    } else {
+      this.setStatus("");
+    }
   }
 
-  renderResults() {
-    const container = this.shadowRoot.querySelector(".results-container");
-    const input = this.shadowRoot.querySelector(".search-input");
+  recentItemsStorageKey() {
+    return "datasette.navigationSearch.recentItems";
+  }
 
-    if (this.matches.length === 0) {
-      const message = input.value.trim()
-        ? "No results found"
-        : "Start typing to search...";
-      container.innerHTML = `<div class="no-results">${message}</div>`;
+  loadRecentItems() {
+    if (typeof localStorage === "undefined") {
+      return [];
+    }
+
+    try {
+      const raw = localStorage.getItem(this.recentItemsStorageKey());
+      if (!raw) {
+        return [];
+      }
+      const parsed = JSON.parse(raw);
+      if (!Array.isArray(parsed)) {
+        return [];
+      }
+      return parsed
+        .filter((item) => item && item.name && item.url)
+        .map((item) => ({
+          name: String(item.name),
+          display_name: item.display_name ? String(item.display_name) : "",
+          url: String(item.url),
+          type: item.type ? String(item.type) : "",
+          description: item.description ? String(item.description) : "",
+        }))
+        .slice(0, 5);
+    } catch (e) {
+      return [];
+    }
+  }
+
+  saveRecentItem(match) {
+    if (
+      typeof localStorage === "undefined" ||
+      !match ||
+      !match.name ||
+      !match.url
+    ) {
       return;
     }
 
-    container.innerHTML = this.matches
-      .map(
-        (match, index) => `
-            <div 
-                class="result-item ${
-                  index === this.selectedIndex ? "selected" : ""
-                }" 
+    try {
+      const item = {
+        name: String(match.name),
+        display_name: match.display_name ? String(match.display_name) : "",
+        url: String(match.url),
+        type: match.type ? String(match.type) : "",
+        description: match.description ? String(match.description) : "",
+      };
+      const recentItems = this.loadRecentItems().filter(
+        (recentItem) => recentItem.url !== item.url,
+      );
+      localStorage.setItem(
+        this.recentItemsStorageKey(),
+        JSON.stringify([item, ...recentItems].slice(0, 5)),
+      );
+    } catch (e) {
+      // localStorage may be unavailable, full, or disabled.
+    }
+  }
+
+  clearRecentItems() {
+    if (typeof localStorage === "undefined") {
+      return;
+    }
+
+    try {
+      localStorage.removeItem(this.recentItemsStorageKey());
+    } catch (e) {
+      localStorage.setItem(this.recentItemsStorageKey(), "[]");
+    }
+    this.renderResults();
+    this.setStatus("Recent items cleared.");
+  }
+
+  jumpSections() {
+    const manager = window.__DATASETTE__;
+    if (!manager || typeof manager.makeJumpSections !== "function") {
+      return [];
+    }
+    const sections = manager.makeJumpSections({
+      navigationSearch: this,
+    });
+    return Array.isArray(sections)
+      ? sections.filter(
+          (section) => section && typeof section.render === "function",
+        )
+      : [];
+  }
+
+  jumpSectionsHtml(jumpSections) {
+    return jumpSections
+      .map((section, index) => {
+        const id = section.id
+          ? ` data-jump-section-id="${this.escapeHtml(section.id)}"`
+          : "";
+        return `<div class="jump-start-content" data-jump-section-index="${index}"${id}></div>`;
+      })
+      .join("");
+  }
+
+  renderJumpSections(container, jumpSections) {
+    jumpSections.forEach((section, index) => {
+      const node = container.querySelector(
+        `[data-jump-section-index="${index}"]`,
+      );
+      if (!node) {
+        return;
+      }
+      section.render(node, {
+        navigationSearch: this,
+        container,
+        input: this.shadowRoot.querySelector(".search-input"),
+      });
+    });
+  }
+
+  resultItemHtml(match, index) {
+    const displayName = match.display_name || match.name;
+    const label =
+      match.display_name && match.display_name !== match.name
+        ? `<div class="result-label">${this.escapeHtml(match.name)}</div>`
+        : "";
+    const type = match.type
+      ? `<div class="result-type">${this.escapeHtml(match.type)}</div>`
+      : "";
+    const description = match.description
+      ? `<div class="result-description">${this.escapeHtml(
+          match.description,
+        )}</div>`
+      : "";
+    return `
+            <div
+                id="${this.resultOptionId(index)}"
+                class="result-item ${index === this.selectedIndex ? "selected" : ""}"
                 data-index="${index}"
                 role="option"
                 aria-selected="${index === this.selectedIndex}"
             >
                 <div>
-                    <div class="result-name">${this.escapeHtml(
-                      match.name,
-                    )}</div>
+                    ${type}
+                    <div class="result-name">${this.escapeHtml(displayName)}</div>
+                    ${label}
                     <div class="result-url">${this.escapeHtml(match.url)}</div>
+                    ${description}
                 </div>
             </div>
-        `,
+        `;
+  }
+
+  renderResults() {
+    const container = this.shadowRoot.querySelector(".results-container");
+    const input = this.shadowRoot.querySelector(".search-input");
+    const showStartContent = !input.value.trim();
+    const jumpSections = showStartContent ? this.jumpSections() : [];
+    const startBlock = showStartContent
+      ? this.jumpSectionsHtml(jumpSections)
+      : "";
+    const recentItems = showStartContent ? this.loadRecentItems() : [];
+    const defaultMatches = showStartContent ? [] : this.matches;
+    const renderedMatches = [...recentItems, ...defaultMatches];
+    this.renderedMatches = renderedMatches;
+    const emptyListbox = `<div id="${this.listboxId}" class="results-list" role="listbox" aria-label="Jump results"></div>`;
+
+    if (renderedMatches.length) {
+      if (
+        this.selectedIndex < 0 ||
+        this.selectedIndex >= renderedMatches.length
+      ) {
+        this.selectedIndex = 0;
+      }
+    } else {
+      this.selectedIndex = -1;
+    }
+
+    if (renderedMatches.length === 0) {
+      if (startBlock) {
+        container.innerHTML = startBlock + emptyListbox;
+        this.renderJumpSections(container, jumpSections);
+      } else if (showStartContent) {
+        container.innerHTML = emptyListbox;
+      } else {
+        const message = input.value.trim()
+          ? "No results found"
+          : "Start typing to search...";
+        container.innerHTML = `${emptyListbox}<div class="no-results">${message}</div>`;
+      }
+      this.updateComboboxState();
+      return;
+    }
+
+    const recentHeading = recentItems.length
+      ? `<div class="results-heading" id="${this.recentHeadingId}">Recent</div>`
+      : "";
+    const recentGroup = recentItems.length
+      ? `<div role="group" aria-labelledby="${this.recentHeadingId}">${recentItems
+          .map((match, index) => this.resultItemHtml(match, index))
+          .join("")}</div>`
+      : "";
+    const recentActions = recentItems.length
+      ? `<div class="recent-actions"><button type="button" class="clear-recent" data-clear-recent-items>Clear recent</button></div>`
+      : "";
+    const defaultHtml = defaultMatches
+      .map((match, index) =>
+        this.resultItemHtml(match, recentItems.length + index),
       )
       .join("");
+    container.innerHTML =
+      startBlock +
+      recentHeading +
+      `<div id="${this.listboxId}" class="results-list" role="listbox" aria-label="Jump results">${recentGroup}${defaultHtml}</div>` +
+      recentActions;
+    this.renderJumpSections(container, jumpSections);
+    this.updateComboboxState();
 
     // Scroll selected item into view
     if (this.selectedIndex >= 0) {
-      const selectedItem = container.children[this.selectedIndex];
+      const selectedItem = container.querySelector(
+        `.result-item[data-index="${this.selectedIndex}"]`,
+      );
       if (selectedItem) {
         selectedItem.scrollIntoView({ block: "nearest" });
       }
@@ -354,22 +816,27 @@ class NavigationSearch extends HTMLElement {
   }
 
   moveSelection(direction) {
+    const matches = this.renderedMatches || this.matches;
     const newIndex = this.selectedIndex + direction;
-    if (newIndex >= 0 && newIndex < this.matches.length) {
+    if (newIndex >= 0 && newIndex < matches.length) {
       this.selectedIndex = newIndex;
       this.renderResults();
     }
   }
 
   selectCurrentItem() {
-    if (this.selectedIndex >= 0 && this.selectedIndex < this.matches.length) {
+    const matches = this.renderedMatches || this.matches;
+    if (this.selectedIndex >= 0 && this.selectedIndex < matches.length) {
       this.selectItem(this.selectedIndex);
     }
   }
 
   selectItem(index) {
-    const match = this.matches[index];
+    const matches = this.renderedMatches || this.matches;
+    const match = matches[index];
     if (match) {
+      this.saveRecentItem(match);
+
       // Dispatch custom event
       this.dispatchEvent(
         new CustomEvent("select", {
@@ -382,32 +849,59 @@ class NavigationSearch extends HTMLElement {
       // Navigate to URL
       window.location.href = match.url;
 
-      this.closeMenu();
+      this.closeMenu({ restoreFocus: false });
     }
   }
 
-  openMenu() {
+  openMenu(trigger) {
     const dialog = this.shadowRoot.querySelector("dialog");
     const input = this.shadowRoot.querySelector(".search-input");
 
-    dialog.showModal();
+    this.restoreFocusTarget = this.focusRestoreTarget(trigger);
+    this.shouldRestoreFocus = true;
+    if (!dialog.open) {
+      dialog.showModal();
+    }
+    this.setNavigationTriggersExpanded(true);
     input.value = "";
     input.focus();
 
-    // Reset state - start with no items shown
+    // Reset state, then populate the default jump list.
     this.matches = [];
     this.selectedIndex = -1;
     this.renderResults();
+    this.setStatus("");
   }
 
-  closeMenu() {
+  closeMenu(options = {}) {
     const dialog = this.shadowRoot.querySelector("dialog");
-    dialog.close();
+    this.shouldRestoreFocus = options.restoreFocus !== false;
+    if (dialog.open) {
+      dialog.close();
+    } else {
+      this.onMenuClosed();
+    }
+  }
+
+  onMenuClosed() {
+    const input = this.shadowRoot.querySelector(".search-input");
+    this.setElementAttribute(input, "aria-expanded", "false");
+    this.removeElementAttribute(input, "aria-activedescendant");
+    this.setNavigationTriggersExpanded(false);
+    this.setStatus("");
+    if (
+      this.shouldRestoreFocus &&
+      this.restoreFocusTarget &&
+      typeof this.restoreFocusTarget.focus === "function"
+    ) {
+      this.restoreFocusTarget.focus();
+    }
+    this.restoreFocusTarget = null;
   }
 
   escapeHtml(text) {
     const div = document.createElement("div");
-    div.textContent = text;
+    div.textContent = text == null ? "" : text;
     return div.innerHTML;
   }
 }

datasette/static/table.js

@@ -1,13 +1,6 @@
 var DROPDOWN_HTML = `<div class="dropdown-menu">
 <div class="hook"></div>
-<ul>
-  <li><a class="dropdown-sort-asc" href="#">Sort ascending</a></li>
-  <li><a class="dropdown-sort-desc" href="#">Sort descending</a></li>
-  <li><a class="dropdown-facet" href="#">Facet by this</a></li>
-  <li><a class="dropdown-hide-column" href="#">Hide this column</a></li>
-  <li><a class="dropdown-show-all-columns" href="#">Show all columns</a></li>
-  <li><a class="dropdown-not-blank" href="#">Show not-blank rows</a></li>
-</ul>
+<ul class="dropdown-actions"></ul>
 <p class="dropdown-column-type"></p>
 <p class="dropdown-column-description"></p>
 </div>`;
@@ -17,54 +10,509 @@ var DROPDOWN_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="14" heig
   <path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1-2-2 2 2 0 0 1 2-2h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 2-2 2 2 0 0 1 2 2v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-.09a1.65 1.65 0 0 0-1.51 1z"></path>
 </svg>`;
 
+var SET_COLUMN_TYPE_DIALOG_ID = "set-column-type-dialog";
+var setColumnTypeDialogState = null;
+
+function getParams() {
+  return new URLSearchParams(location.search);
+}
+
+function paramsToUrl(params) {
+  var s = params.toString();
+  return s ? "?" + s : location.pathname;
+}
+
+function sortDescUrl(column) {
+  var params = getParams();
+  params.set("_sort_desc", column);
+  params.delete("_sort");
+  params.delete("_next");
+  return paramsToUrl(params);
+}
+
+function sortAscUrl(column) {
+  var params = getParams();
+  params.set("_sort", column);
+  params.delete("_sort_desc");
+  params.delete("_next");
+  return paramsToUrl(params);
+}
+
+function facetUrl(column) {
+  var params = getParams();
+  params.append("_facet", column);
+  return paramsToUrl(params);
+}
+
+function hideColumnUrl(column) {
+  var params = getParams();
+  params.append("_nocol", column);
+  return paramsToUrl(params);
+}
+
+function showAllColumnsUrl() {
+  var params = getParams();
+  params.delete("_nocol");
+  params.delete("_col");
+  return paramsToUrl(params);
+}
+
+function notBlankUrl(column) {
+  var params = getParams();
+  params.set(`${column}__notblank`, "1");
+  return paramsToUrl(params);
+}
+
+function getDisplayedFacets() {
+  return Array.from(document.querySelectorAll(".facet-info")).map(
+    (el) => el.dataset.column,
+  );
+}
+
+function getColumnClassName(th) {
+  return Array.from(th.classList).find((className) =>
+    className.startsWith("col-"),
+  );
+}
+
+function getColumnCells(th) {
+  var table = th.closest("table");
+  var columnClassName = getColumnClassName(th);
+  if (!table || !columnClassName) {
+    return [];
+  }
+  return Array.from(table.querySelectorAll("td." + columnClassName));
+}
+
+function getColumnMeta(th) {
+  return {
+    columnName: th.dataset.column,
+    columnNotNull: th.dataset.columnNotNull === "1",
+    columnType: th.dataset.columnType,
+    isPk: th.dataset.isPk === "1",
+  };
+}
+
+function getColumnTypeText(th) {
+  var columnType = th.dataset.columnType;
+  if (!columnType) {
+    return null;
+  }
+  var notNull = th.dataset.columnNotNull === "1" ? " NOT NULL" : "";
+  return `Type: ${columnType.toUpperCase()}${notNull}`;
+}
+
+function getSetColumnTypeData() {
+  return window._setColumnTypeData || null;
+}
+
+function getSetColumnTypeConfig(column) {
+  var data = getSetColumnTypeData();
+  if (!data || !data.columns) {
+    return null;
+  }
+  return data.columns[column] || null;
+}
+
+function canSetColumnType() {
+  return !!(getSetColumnTypeData() && window.HTMLDialogElement && window.fetch);
+}
+
+function setColumnTypeActionLabel(column) {
+  var columnConfig = getSetColumnTypeConfig(column);
+  if (!columnConfig) {
+    return null;
+  }
+  return columnConfig.current
+    ? `Custom type: ${columnConfig.current.type}`
+    : "Set custom type";
+}
+
+function createSetColumnTypeOption(value, name, description, checked) {
+  var label = document.createElement("label");
+  label.className = "set-column-type-option";
+
+  var input = document.createElement("input");
+  input.type = "radio";
+  input.name = "set-column-type-choice";
+  input.value = value;
+  input.checked = checked;
+
+  var content = document.createElement("span");
+  content.className = "set-column-type-option-content";
+
+  var title = document.createElement("span");
+  title.className = "set-column-type-option-name";
+  title.textContent = name;
+
+  var detail = document.createElement("span");
+  detail.className = "set-column-type-option-description";
+  detail.textContent = description;
+
+  content.appendChild(title);
+  content.appendChild(detail);
+  label.appendChild(input);
+  label.appendChild(content);
+  return label;
+}
+
+function setSetColumnTypeDialogBusy(state, isBusy) {
+  state.isBusy = isBusy;
+  state.saveButton.disabled = isBusy;
+  state.cancelButton.disabled = isBusy;
+  Array.from(
+    state.optionsWrap.querySelectorAll('input[name="set-column-type-choice"]'),
+  ).forEach(function (input) {
+    input.disabled = isBusy;
+  });
+  state.saveButton.textContent = isBusy ? "Saving..." : "Save";
+}
+
+function clearSetColumnTypeDialogError(state) {
+  state.error.hidden = true;
+  state.error.textContent = "";
+}
+
+function showSetColumnTypeDialogError(state, message) {
+  state.error.hidden = false;
+  state.error.textContent = message;
+}
+
+function ensureSetColumnTypeDialog() {
+  if (setColumnTypeDialogState) {
+    return setColumnTypeDialogState;
+  }
+  if (!window.HTMLDialogElement) {
+    return null;
+  }
+
+  var dialog = document.createElement("dialog");
+  dialog.id = SET_COLUMN_TYPE_DIALOG_ID;
+  dialog.className = "set-column-type-dialog";
+  dialog.setAttribute("aria-labelledby", "set-column-type-title");
+  dialog.innerHTML = `
+    <div class="modal-header">
+      <span class="modal-title" id="set-column-type-title">Set custom type</span>
+      <span class="modal-meta"></span>
+    </div>
+    <p class="set-column-type-status"></p>
+    <p class="set-column-type-error" hidden></p>
+    <div class="set-column-type-options"></div>
+    <div class="modal-footer">
+      <span class="footer-info"></span>
+      <button type="button" class="btn btn-ghost set-column-type-cancel">Cancel</button>
+      <button type="button" class="btn btn-primary set-column-type-save">Save</button>
+    </div>
+  `;
+  document.body.appendChild(dialog);
+
+  setColumnTypeDialogState = {
+    dialog: dialog,
+    meta: dialog.querySelector(".modal-meta"),
+    status: dialog.querySelector(".set-column-type-status"),
+    error: dialog.querySelector(".set-column-type-error"),
+    optionsWrap: dialog.querySelector(".set-column-type-options"),
+    footerInfo: dialog.querySelector(".footer-info"),
+    cancelButton: dialog.querySelector(".set-column-type-cancel"),
+    saveButton: dialog.querySelector(".set-column-type-save"),
+    currentColumn: null,
+    currentConfig: null,
+    isBusy: false,
+  };
+
+  setColumnTypeDialogState.cancelButton.addEventListener("click", function () {
+    if (!setColumnTypeDialogState.isBusy) {
+      dialog.close();
+    }
+  });
+
+  dialog.addEventListener("click", function (ev) {
+    if (ev.target === dialog && !setColumnTypeDialogState.isBusy) {
+      dialog.close();
+    }
+  });
+
+  dialog.addEventListener("cancel", function (ev) {
+    if (setColumnTypeDialogState.isBusy) {
+      ev.preventDefault();
+    }
+  });
+
+  dialog.addEventListener("close", function () {
+    clearSetColumnTypeDialogError(setColumnTypeDialogState);
+    setSetColumnTypeDialogBusy(setColumnTypeDialogState, false);
+  });
+
+  setColumnTypeDialogState.saveButton.addEventListener("click", async function () {
+    var state = setColumnTypeDialogState;
+    var selected = state.dialog.querySelector(
+      'input[name="set-column-type-choice"]:checked',
+    );
+    var selectedType = selected ? selected.value : "";
+    var currentType = state.currentConfig.current
+      ? state.currentConfig.current.type
+      : "";
+
+    if (selectedType === currentType) {
+      state.dialog.close();
+      return;
+    }
+
+    clearSetColumnTypeDialogError(state);
+    setSetColumnTypeDialogBusy(state, true);
+
+    var payload = {
+      column: state.currentColumn,
+      column_type: selectedType ? { type: selectedType } : null,
+    };
+
+    try {
+      var response = await fetch(getSetColumnTypeData().path, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Accept: "application/json",
+        },
+        body: JSON.stringify(payload),
+      });
+      var data = await response.json();
+      if (!response.ok || data.ok === false) {
+        var message = (data.errors || ["Request failed"]).join(" ");
+        throw new Error(message);
+      }
+      location.reload();
+    } catch (error) {
+      setSetColumnTypeDialogBusy(state, false);
+      showSetColumnTypeDialogError(state, error.message || "Request failed");
+    }
+  });
+
+  return setColumnTypeDialogState;
+}
+
+function openSetColumnTypeDialog(th) {
+  var column = th.dataset.column;
+  var columnConfig = getSetColumnTypeConfig(column);
+  if (!columnConfig) {
+    return;
+  }
+
+  var state = ensureSetColumnTypeDialog();
+  if (!state) {
+    return;
+  }
+
+  clearSetColumnTypeDialogError(state);
+  setSetColumnTypeDialogBusy(state, false);
+  state.currentColumn = column;
+  state.currentConfig = columnConfig;
+  state.status.textContent = `Column: ${column}`;
+  state.meta.textContent = getColumnTypeText(th) || "Type unavailable";
+  state.footerInfo.textContent = columnConfig.current
+    ? `Current custom type: ${columnConfig.current.type}`
+    : "No custom type set.";
+  state.optionsWrap.innerHTML = "";
+
+  var currentType = columnConfig.current ? columnConfig.current.type : "";
+  state.optionsWrap.appendChild(
+    createSetColumnTypeOption(
+      "",
+      "No custom type",
+      "Use standard Datasette rendering without a custom type.",
+      currentType === "",
+    ),
+  );
+
+  columnConfig.options.forEach(function (option) {
+    state.optionsWrap.appendChild(
+      createSetColumnTypeOption(
+        option.name,
+        option.name,
+        option.description,
+        option.name === currentType,
+      ),
+    );
+  });
+
+  if (!columnConfig.options.length) {
+    var emptyState = document.createElement("p");
+    emptyState.className = "set-column-type-empty";
+    emptyState.textContent =
+      "No registered custom types are compatible with this SQLite type.";
+    state.optionsWrap.appendChild(emptyState);
+  }
+
+  if (!state.dialog.open) {
+    state.dialog.showModal();
+  }
+  var selectedOption = state.dialog.querySelector(
+    'input[name="set-column-type-choice"]:checked',
+  );
+  if (selectedOption) {
+    selectedOption.focus();
+  } else {
+    state.saveButton.focus();
+  }
+}
+
+function canChooseColumns() {
+  return !!(
+    document.querySelector("column-chooser") && window._columnChooserData
+  );
+}
+
+function shouldShowShowAllColumns() {
+  var params = getParams();
+  return params.getAll("_nocol").length || params.getAll("_col").length;
+}
+
+function hasMultipleVisibleColumns(manager) {
+  return (
+    Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter(
+      (th) => th.dataset.column && th.dataset.isLinkColumn !== "1",
+    ).length > 1
+  );
+}
+
+function buildColumnActionItems(manager, th, options) {
+  options = options || {};
+  var params = getParams();
+  var column = th.dataset.column;
+  var columnActions = [];
+  var isSortable = !!th.querySelector("a");
+  var isFirstColumn = th.parentElement.querySelector("th:first-of-type") === th;
+  var isSinglePk =
+    th.dataset.isPk === "1" &&
+    document.querySelectorAll('th[data-is-pk="1"]').length === 1;
+  var hasBlankValues = getColumnCells(th).some(
+    (el) => el.innerText.trim() === "",
+  );
+
+  if (isSortable && params.get("_sort") !== column) {
+    columnActions.push({
+      label: "Sort ascending",
+      href: sortAscUrl(column),
+    });
+  }
+
+  if (isSortable && params.get("_sort_desc") !== column) {
+    columnActions.push({
+      label: "Sort descending",
+      href: sortDescUrl(column),
+    });
+  }
+
+  if (
+    DATASETTE_ALLOW_FACET &&
+    !isFirstColumn &&
+    !getDisplayedFacets().includes(column) &&
+    !isSinglePk
+  ) {
+    columnActions.push({
+      label: "Facet by this",
+      href: facetUrl(column),
+    });
+  }
+
+  if (options.includeChooseColumns && canChooseColumns()) {
+    columnActions.push({
+      label: "Choose columns",
+      href: "#",
+      onClick:
+        options.onChooseColumns ||
+        function (ev) {
+          ev.preventDefault();
+          openColumnChooser();
+        },
+    });
+  }
+
+  if (canSetColumnType() && getSetColumnTypeConfig(column)) {
+    columnActions.push({
+      label: setColumnTypeActionLabel(column),
+      href: "#",
+      onClick:
+        options.onSetColumnType ||
+        function (ev) {
+          ev.preventDefault();
+          window.setTimeout(function () {
+            openSetColumnTypeDialog(th);
+          }, 0);
+        },
+    });
+  }
+
+  if (th.dataset.isPk !== "1" && hasMultipleVisibleColumns(manager)) {
+    columnActions.push({
+      label: "Hide this column",
+      href: hideColumnUrl(column),
+    });
+  }
+
+  if (options.includeShowAllColumns && shouldShowShowAllColumns()) {
+    columnActions.push({
+      label: "Show all columns",
+      href: showAllColumnsUrl(),
+    });
+  }
+
+  if (params.get(`${column}__notblank`) !== "1" && hasBlankValues) {
+    columnActions.push({
+      label: "Show not-blank rows",
+      href: notBlankUrl(column),
+    });
+  }
+
+  return columnActions.concat(manager.makeColumnActions(getColumnMeta(th)));
+}
+
+function buildColumnActionState(manager, th, options) {
+  return {
+    column: th.dataset.column,
+    columnDescription: th.dataset.columnDescription || null,
+    columnMeta: getColumnMeta(th),
+    columnTypeText: getColumnTypeText(th),
+    actionItems: buildColumnActionItems(manager, th, options),
+  };
+}
+
+function initializeColumnActions(manager) {
+  manager.columnActions = {
+    buildColumnActionState: function (th, options) {
+      return buildColumnActionState(manager, th, options);
+    },
+    buildColumnActionItems: function (th, options) {
+      return buildColumnActionItems(manager, th, options);
+    },
+    canChooseColumns: canChooseColumns,
+    facetUrl: facetUrl,
+    getColumnMeta: getColumnMeta,
+    getColumnTypeText: getColumnTypeText,
+    hideColumnUrl: hideColumnUrl,
+    notBlankUrl: notBlankUrl,
+    shouldShowShowAllColumns: shouldShowShowAllColumns,
+    showAllColumnsUrl: showAllColumnsUrl,
+    sortAscUrl: sortAscUrl,
+    sortDescUrl: sortDescUrl,
+  };
+}
+
+function renderActionLink(itemConfig) {
+  var newLink = document.createElement("a");
+  newLink.textContent = itemConfig.label;
+  newLink.href = itemConfig.href || "#";
+  if (itemConfig.onClick) {
+    newLink.addEventListener("click", itemConfig.onClick);
+  }
+  return newLink;
+}
+
 /** Main initialization function for Datasette Table interactions */
 const initDatasetteTable = function (manager) {
   // Feature detection
   if (!window.URLSearchParams) {
     return;
   }
-  function getParams() {
-    return new URLSearchParams(location.search);
-  }
-  function paramsToUrl(params) {
-    var s = params.toString();
-    return s ? "?" + s : location.pathname;
-  }
-  function sortDescUrl(column) {
-    var params = getParams();
-    params.set("_sort_desc", column);
-    params.delete("_sort");
-    params.delete("_next");
-    return paramsToUrl(params);
-  }
-  function sortAscUrl(column) {
-    var params = getParams();
-    params.set("_sort", column);
-    params.delete("_sort_desc");
-    params.delete("_next");
-    return paramsToUrl(params);
-  }
-  function facetUrl(column) {
-    var params = getParams();
-    params.append("_facet", column);
-    return paramsToUrl(params);
-  }
-  function hideColumnUrl(column) {
-    var params = getParams();
-    params.append("_nocol", column);
-    return paramsToUrl(params);
-  }
-  function showAllColumnsUrl() {
-    var params = getParams();
-    params.delete("_nocol");
-    params.delete("_col");
-    return paramsToUrl(params);
-  }
-  function notBlankUrl(column) {
-    var params = getParams();
-    params.set(`${column}__notblank`, "1");
-    return paramsToUrl(params);
-  }
   function closeMenu() {
     menu.style.display = "none";
     menu.classList.remove("anim-scale-in");
@@ -96,87 +544,41 @@ const initDatasetteTable = function (manager) {
     var rect = th.getBoundingClientRect();
     var menuTop = rect.bottom + window.scrollY;
     var menuLeft = rect.left + window.scrollX;
-    var column = th.getAttribute("data-column");
-    var params = getParams();
-    var sort = menu.querySelector("a.dropdown-sort-asc");
-    var sortDesc = menu.querySelector("a.dropdown-sort-desc");
-    var facetItem = menu.querySelector("a.dropdown-facet");
-    var notBlank = menu.querySelector("a.dropdown-not-blank");
-    var hideColumn = menu.querySelector("a.dropdown-hide-column");
-    var showAllColumns = menu.querySelector("a.dropdown-show-all-columns");
-    if (params.get("_sort") == column) {
-      sort.parentNode.style.display = "none";
-    } else {
-      sort.parentNode.style.display = "block";
-      sort.setAttribute("href", sortAscUrl(column));
-    }
-    if (params.get("_sort_desc") == column) {
-      sortDesc.parentNode.style.display = "none";
-    } else {
-      sortDesc.parentNode.style.display = "block";
-      sortDesc.setAttribute("href", sortDescUrl(column));
-    }
-    /* Show hide columns options */
-    if (params.get("_nocol") || params.get("_col")) {
-      showAllColumns.parentNode.style.display = "block";
-      showAllColumns.setAttribute("href", showAllColumnsUrl());
-    } else {
-      showAllColumns.parentNode.style.display = "none";
-    }
-    if (th.getAttribute("data-is-pk") != "1") {
-      hideColumn.parentNode.style.display = "block";
-      hideColumn.setAttribute("href", hideColumnUrl(column));
-    } else {
-      hideColumn.parentNode.style.display = "none";
-    }
-    /* Only show "Facet by this" if it's not the first column, not selected,
-       not a single PK and the Datasette allow_facet setting is True */
-    var displayedFacets = Array.from(
-      document.querySelectorAll(".facet-info"),
-    ).map((el) => el.dataset.column);
-    var isFirstColumn =
-      th.parentElement.querySelector("th:first-of-type") == th;
-    var isSinglePk =
-      th.getAttribute("data-is-pk") == "1" &&
-      document.querySelectorAll('th[data-is-pk="1"]').length == 1;
-    if (
-      !DATASETTE_ALLOW_FACET ||
-      isFirstColumn ||
-      displayedFacets.includes(column) ||
-      isSinglePk
-    ) {
-      facetItem.parentNode.style.display = "none";
-    } else {
-      facetItem.parentNode.style.display = "block";
-      facetItem.setAttribute("href", facetUrl(column));
-    }
-    /* Show notBlank option if not selected AND at least one visible blank value */
-    var tdsForThisColumn = Array.from(
-      th.closest("table").querySelectorAll("td." + th.className),
-    );
-    if (
-      params.get(`${column}__notblank`) != "1" &&
-      tdsForThisColumn.filter((el) => el.innerText.trim() == "").length
-    ) {
-      notBlank.parentNode.style.display = "block";
-      notBlank.setAttribute("href", notBlankUrl(column));
-    } else {
-      notBlank.parentNode.style.display = "none";
-    }
-    var columnTypeP = menu.querySelector(".dropdown-column-type");
-    var columnType = th.dataset.columnType;
-    var notNull = th.dataset.columnNotNull == 1 ? " NOT NULL" : "";
+    var actionState = manager.columnActions.buildColumnActionState(th, {
+      includeChooseColumns: true,
+      includeShowAllColumns: true,
+      onChooseColumns: function (ev) {
+        ev.preventDefault();
+        closeMenu();
+        openColumnChooser();
+      },
+      onSetColumnType: function (ev) {
+        ev.preventDefault();
+        closeMenu();
+        window.setTimeout(function () {
+          openSetColumnTypeDialog(th);
+        }, 0);
+      },
+    });
+    var menuList = menu.querySelector("ul.dropdown-actions");
+    menuList.innerHTML = "";
+    actionState.actionItems.forEach((itemConfig) => {
+      var menuItem = document.createElement("li");
+      menuItem.appendChild(renderActionLink(itemConfig));
+      menuList.appendChild(menuItem);
+    });
 
-    if (columnType) {
+    var columnTypeP = menu.querySelector(".dropdown-column-type");
+    if (actionState.columnTypeText) {
       columnTypeP.style.display = "block";
-      columnTypeP.innerText = `Type: ${columnType.toUpperCase()}${notNull}`;
+      columnTypeP.innerText = actionState.columnTypeText;
     } else {
       columnTypeP.style.display = "none";
     }
 
     var columnDescriptionP = menu.querySelector(".dropdown-column-description");
-    if (th.dataset.columnDescription) {
-      columnDescriptionP.innerText = th.dataset.columnDescription;
+    if (actionState.columnDescription) {
+      columnDescriptionP.innerText = actionState.columnDescription;
       columnDescriptionP.style.display = "block";
     } else {
       columnDescriptionP.style.display = "none";
@@ -187,39 +589,6 @@ const initDatasetteTable = function (manager) {
     menu.style.display = "block";
     menu.classList.add("anim-scale-in");
 
-    // Custom menu items on each render
-    // Plugin hook: allow adding JS-based additional menu items
-    const columnActionsPayload = {
-      columnName: th.dataset.column,
-      columnNotNull: th.dataset.columnNotNull === "1",
-      columnType: th.dataset.columnType,
-      isPk: th.dataset.isPk === "1",
-    };
-    const columnItemConfigs = manager.makeColumnActions(columnActionsPayload);
-
-    const menuList = menu.querySelector("ul");
-    columnItemConfigs.forEach((itemConfig) => {
-      // Remove items from previous render. We assume entries have unique labels.
-      const existingItems = menuList.querySelectorAll(`li`);
-      Array.from(existingItems)
-        .filter((item) => item.innerText === itemConfig.label)
-        .forEach((node) => {
-          node.remove();
-        });
-
-      const newLink = document.createElement("a");
-      newLink.textContent = itemConfig.label;
-      newLink.href = itemConfig.href ?? "#";
-      if (itemConfig.onClick) {
-        newLink.onclick = itemConfig.onClick;
-      }
-
-      // Attach new elements to DOM
-      const menuItem = document.createElement("li");
-      menuItem.appendChild(newLink);
-      menuList.appendChild(menuItem);
-    });
-
     // Measure width of menu and adjust position if too far right
     const menuWidth = menu.offsetWidth;
     const windowWidth = window.innerWidth;
@@ -330,10 +699,55 @@ function initAutocompleteForFilterValues(manager) {
   });
 }
 
+/** Open the column-chooser web component */
+function openColumnChooser() {
+  var chooser = document.querySelector("column-chooser");
+  var data = window._columnChooserData;
+  if (!chooser || !data) return;
+
+  var nonPkColumns = data.allColumns.filter(function (col) {
+    return data.primaryKeys.indexOf(col) === -1;
+  });
+  var selected = data.selectedColumns.filter(function (col) {
+    return data.primaryKeys.indexOf(col) === -1;
+  });
+
+  chooser.open({
+    columns: nonPkColumns,
+    selected: selected,
+    onApply: function (cols) {
+      var params = new URLSearchParams(location.search);
+      params.delete("_col");
+      params.delete("_nocol");
+      params.delete("_next");
+
+      if (cols.length === nonPkColumns.length) {
+        // Check if order matches original - if so, no params needed
+        var orderMatches = cols.every(function (col, i) {
+          return col === nonPkColumns[i];
+        });
+        if (!orderMatches) {
+          cols.forEach(function (col) {
+            params.append("_col", col);
+          });
+        }
+      } else {
+        cols.forEach(function (col) {
+          params.append("_col", col);
+        });
+      }
+      var qs = params.toString();
+      location.href = qs ? "?" + qs : location.pathname;
+    },
+  });
+}
+
 // Ensures Table UI is initialized only after the Manager is ready.
 document.addEventListener("datasette_init", function (evt) {
   const { detail: manager } = evt;
 
+  initializeColumnActions(manager);
+
   // Main table
   initDatasetteTable(manager);
 

datasette/stored_queries.py

@@ -0,0 +1,623 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+from typing import Any, Iterable
+
+from .resources import TableResource
+from .utils import named_parameters, sqlite3, tilde_encode, urlsafe_components
+from .utils.asgi import Forbidden
+
+UNCHANGED = object()
+
+QUERY_OPTION_FIELDS = (
+    "hide_sql",
+    "fragment",
+    "on_success_message",
+    "on_success_message_sql",
+    "on_success_redirect",
+    "on_error_message",
+    "on_error_redirect",
+)
+
+
+@dataclass
+class StoredQuery:
+    database: str
+    name: str
+    sql: str
+    title: str | None
+    description: str | None
+    description_html: str | None
+    hide_sql: bool
+    fragment: str | None
+    parameters: list[str]
+    is_write: bool
+    is_private: bool
+    is_trusted: bool
+    source: str
+    owner_id: str | None
+    on_success_message: str | None
+    on_success_message_sql: str | None
+    on_success_redirect: str | None
+    on_error_message: str | None
+    on_error_redirect: str | None
+    private: bool | None = None
+
+
+@dataclass
+class StoredQueryPage:
+    queries: list[StoredQuery]
+    next: str | None
+    has_more: bool
+    limit: int
+
+
+def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]:
+    data = {
+        "database": query.database,
+        "name": query.name,
+        "sql": query.sql,
+        "title": query.title,
+        "description": query.description,
+        "description_html": query.description_html,
+        "hide_sql": query.hide_sql,
+        "fragment": query.fragment,
+        "params": list(query.parameters),
+        "parameters": list(query.parameters),
+        "is_write": query.is_write,
+        "is_private": query.is_private,
+        "is_trusted": query.is_trusted,
+        "source": query.source,
+        "owner_id": query.owner_id,
+        "on_success_message": query.on_success_message,
+        "on_success_message_sql": query.on_success_message_sql,
+        "on_success_redirect": query.on_success_redirect,
+        "on_error_message": query.on_error_message,
+        "on_error_redirect": query.on_error_redirect,
+    }
+    if query.private is not None:
+        data["private"] = query.private
+    return data
+
+
+def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]:
+    return {
+        "queries": [stored_query_to_dict(query) for query in page.queries],
+        "next": page.next,
+        "has_more": page.has_more,
+        "limit": page.limit,
+    }
+
+
+async def save_queries_from_config(datasette: Any) -> None:
+    # Apply configured query entries from datasette.yaml to the internal table.
+    await datasette.get_internal_database().execute_write(
+        "DELETE FROM queries WHERE source = 'config'"
+    )
+    for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items():
+        for query_name, query_config in (db_config.get("queries") or {}).items():
+            if not isinstance(query_config, dict):
+                query_config = {"sql": query_config}
+            await datasette.add_query(
+                dbname,
+                query_name,
+                query_config["sql"],
+                title=query_config.get("title"),
+                description=query_config.get("description"),
+                description_html=query_config.get("description_html"),
+                hide_sql=bool(query_config.get("hide_sql")),
+                fragment=query_config.get("fragment"),
+                parameters=query_config.get("params"),
+                is_write=bool(query_config.get("write")),
+                is_private=bool(query_config.get("is_private")),
+                is_trusted=bool(query_config.get("is_trusted", True)),
+                source="config",
+                on_success_message=query_config.get("on_success_message"),
+                on_success_message_sql=query_config.get("on_success_message_sql"),
+                on_success_redirect=query_config.get("on_success_redirect"),
+                on_error_message=query_config.get("on_error_message"),
+                on_error_redirect=query_config.get("on_error_redirect"),
+            )
+
+
+def query_row_to_stored_query(
+    row: Any, private: bool | None = None
+) -> StoredQuery | None:
+    if row is None:
+        return None
+    parameters = json.loads(row["parameters"] or "[]")
+    options = json.loads(row["options"] or "{}")
+    return StoredQuery(
+        database=row["database_name"],
+        name=row["name"],
+        sql=row["sql"],
+        title=row["title"],
+        description=row["description"],
+        description_html=row["description_html"],
+        hide_sql=bool(options.get("hide_sql")),
+        fragment=options.get("fragment"),
+        parameters=parameters,
+        is_write=bool(row["is_write"]),
+        is_private=bool(row["is_private"]),
+        is_trusted=bool(row["is_trusted"]),
+        source=row["source"],
+        owner_id=row["owner_id"],
+        on_success_message=options.get("on_success_message"),
+        on_success_message_sql=options.get("on_success_message_sql"),
+        on_success_redirect=options.get("on_success_redirect"),
+        on_error_message=options.get("on_error_message"),
+        on_error_redirect=options.get("on_error_redirect"),
+        private=private,
+    )
+
+
+def query_options_json(options: dict[str, Any]) -> str:
+    options_dict = {}
+    for field in QUERY_OPTION_FIELDS:
+        value = options.get(field)
+        if field == "hide_sql":
+            if value:
+                options_dict[field] = True
+        elif value is not None:
+            options_dict[field] = value
+    return json.dumps(options_dict, sort_keys=True)
+
+
+async def add_query(
+    datasette: Any,
+    database: str,
+    name: str,
+    sql: str,
+    *,
+    title: str | None = None,
+    description: str | None = None,
+    description_html: str | None = None,
+    hide_sql: bool = False,
+    fragment: str | None = None,
+    parameters: Iterable[str] | None = None,
+    is_write: bool = False,
+    is_private: bool = False,
+    is_trusted: bool = False,
+    source: str = "plugin",
+    owner_id: str | None = None,
+    on_success_message: str | None = None,
+    on_success_message_sql: str | None = None,
+    on_success_redirect: str | None = None,
+    on_error_message: str | None = None,
+    on_error_redirect: str | None = None,
+    replace: bool = True,
+) -> None:
+    parameters_json = json.dumps(list(parameters or []))
+    options_json = query_options_json(
+        {
+            "hide_sql": hide_sql,
+            "fragment": fragment,
+            "on_success_message": on_success_message,
+            "on_success_message_sql": on_success_message_sql,
+            "on_success_redirect": on_success_redirect,
+            "on_error_message": on_error_message,
+            "on_error_redirect": on_error_redirect,
+        }
+    )
+    sql_statement = """
+        INSERT INTO queries (
+            database_name, name, sql, title, description, description_html,
+            options, parameters, is_write, is_private, is_trusted, source, owner_id
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    """
+    if replace:
+        sql_statement += """
+            ON CONFLICT(database_name, name) DO UPDATE SET
+                sql = excluded.sql,
+                title = excluded.title,
+                description = excluded.description,
+                description_html = excluded.description_html,
+                options = excluded.options,
+                parameters = excluded.parameters,
+                is_write = excluded.is_write,
+                is_private = excluded.is_private,
+                is_trusted = excluded.is_trusted,
+                source = excluded.source,
+                owner_id = excluded.owner_id,
+                updated_at = CURRENT_TIMESTAMP
+        """
+    await datasette.get_internal_database().execute_write(
+        sql_statement,
+        [
+            database,
+            name,
+            sql,
+            title,
+            description,
+            description_html,
+            options_json,
+            parameters_json,
+            int(bool(is_write)),
+            int(bool(is_private)),
+            int(bool(is_trusted)),
+            source,
+            owner_id,
+        ],
+    )
+
+
+async def update_query(
+    datasette: Any,
+    database: str,
+    name: str,
+    *,
+    sql=UNCHANGED,
+    title=UNCHANGED,
+    description=UNCHANGED,
+    description_html=UNCHANGED,
+    hide_sql=UNCHANGED,
+    fragment=UNCHANGED,
+    parameters=UNCHANGED,
+    is_write=UNCHANGED,
+    is_private=UNCHANGED,
+    is_trusted=UNCHANGED,
+    source=UNCHANGED,
+    owner_id=UNCHANGED,
+    on_success_message=UNCHANGED,
+    on_success_message_sql=UNCHANGED,
+    on_success_redirect=UNCHANGED,
+    on_error_message=UNCHANGED,
+    on_error_redirect=UNCHANGED,
+) -> None:
+    fields = {
+        "sql": sql,
+        "title": title,
+        "description": description,
+        "description_html": description_html,
+        "parameters": parameters,
+        "is_write": is_write,
+        "is_private": is_private,
+        "is_trusted": is_trusted,
+        "source": source,
+        "owner_id": owner_id,
+    }
+    option_fields = {
+        "hide_sql": hide_sql,
+        "fragment": fragment,
+        "on_success_message": on_success_message,
+        "on_success_message_sql": on_success_message_sql,
+        "on_success_redirect": on_success_redirect,
+        "on_error_message": on_error_message,
+        "on_error_redirect": on_error_redirect,
+    }
+    updates = []
+    params = []
+    for field, value in fields.items():
+        if value is UNCHANGED:
+            continue
+        if field in {"is_write", "is_private", "is_trusted"}:
+            value = int(bool(value))
+        elif field == "parameters":
+            value = json.dumps(list(value or []))
+        updates.append(f"{field} = ?")
+        params.append(value)
+    changed_options = {
+        field: value for field, value in option_fields.items() if value is not UNCHANGED
+    }
+    if changed_options:
+        rows = await datasette.get_internal_database().execute(
+            """
+            SELECT options FROM queries
+            WHERE database_name = ? AND name = ?
+            """,
+            [database, name],
+        )
+        row = rows.first()
+        options = json.loads(row["options"] or "{}") if row is not None else {}
+        for field, value in changed_options.items():
+            if field == "hide_sql":
+                if value:
+                    options[field] = True
+                else:
+                    options.pop(field, None)
+            elif value is None:
+                options.pop(field, None)
+            else:
+                options[field] = value
+        updates.append("options = ?")
+        params.append(json.dumps(options, sort_keys=True))
+    if not updates:
+        return
+    updates.append("updated_at = CURRENT_TIMESTAMP")
+    params.extend([database, name])
+    await datasette.get_internal_database().execute_write(
+        """
+        UPDATE queries
+        SET {}
+        WHERE database_name = ? AND name = ?
+        """.format(", ".join(updates)),
+        params,
+    )
+
+
+async def remove_query(
+    datasette: Any, database: str, name: str, source: str | None = None
+) -> None:
+    sql = "DELETE FROM queries WHERE database_name = ? AND name = ?"
+    params = [database, name]
+    if source is not None:
+        sql += " AND source = ?"
+        params.append(source)
+    await datasette.get_internal_database().execute_write(sql, params)
+
+
+async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None:
+    rows = await datasette.get_internal_database().execute(
+        """
+        SELECT * FROM queries
+        WHERE database_name = ? AND name = ?
+        """,
+        [database, name],
+    )
+    return query_row_to_stored_query(rows.first())
+
+
+async def count_queries(
+    datasette: Any,
+    database: str | None = None,
+    *,
+    actor: dict[str, Any] | None = None,
+    q: str | None = None,
+    is_write: bool | None = None,
+    is_private: bool | None = None,
+    is_trusted: bool | None = None,
+    source: str | None = None,
+    owner_id: str | None = None,
+) -> int:
+    allowed_sql, allowed_params = await datasette.allowed_resources_sql(
+        action="view-query",
+        actor=actor,
+        parent=database,
+    )
+    params = dict(allowed_params)
+    where_clauses = []
+    if database is not None:
+        params["query_database"] = database
+        where_clauses.append("q.database_name = :query_database")
+
+    if q:
+        where_clauses.append("""
+            (
+                q.name LIKE :query_search
+                OR q.title LIKE :query_search
+                OR q.description LIKE :query_search
+                OR q.sql LIKE :query_search
+            )
+            """)
+        params["query_search"] = "%{}%".format(q)
+    if is_write is not None:
+        where_clauses.append("q.is_write = :query_is_write")
+        params["query_is_write"] = int(bool(is_write))
+    if is_private is not None:
+        where_clauses.append("q.is_private = :query_is_private")
+        params["query_is_private"] = int(bool(is_private))
+    if is_trusted is not None:
+        where_clauses.append("q.is_trusted = :query_is_trusted")
+        params["query_is_trusted"] = int(bool(is_trusted))
+    if source is not None:
+        where_clauses.append("q.source = :query_source")
+        params["query_source"] = source
+    if owner_id is not None:
+        where_clauses.append("q.owner_id = :query_owner_id")
+        params["query_owner_id"] = owner_id
+
+    row = (
+        await datasette.get_internal_database().execute(
+            """
+            SELECT count(*) AS count
+            FROM queries q
+            JOIN (
+                {allowed_sql}
+            ) allowed
+              ON allowed.parent = q.database_name
+             AND allowed.child = q.name
+            WHERE {where}
+            """.format(
+                allowed_sql=allowed_sql,
+                where=" AND ".join(where_clauses) or "1 = 1",
+            ),
+            params,
+        )
+    ).first()
+    return row["count"]
+
+
+async def list_queries(
+    datasette: Any,
+    database: str | None = None,
+    *,
+    actor: dict[str, Any] | None = None,
+    limit: int = 50,
+    cursor: str | None = None,
+    q: str | None = None,
+    is_write: bool | None = None,
+    is_private: bool | None = None,
+    is_trusted: bool | None = None,
+    source: str | None = None,
+    owner_id: str | None = None,
+    include_private: bool = False,
+) -> StoredQueryPage:
+    limit = min(max(1, int(limit)), 1000)
+    allowed_sql, allowed_params = await datasette.allowed_resources_sql(
+        action="view-query",
+        actor=actor,
+        parent=database,
+        include_is_private=include_private,
+    )
+    params = dict(allowed_params)
+    params.update({"limit": limit + 1})
+    sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))"
+    where_clauses = []
+    order_by = "q.database_name, sort_key, q.name"
+    if database is not None:
+        params["query_database"] = database
+        where_clauses.append("q.database_name = :query_database")
+        order_by = "sort_key, q.name"
+
+    if cursor:
+        try:
+            components = urlsafe_components(cursor)
+        except ValueError:
+            components = []
+        if database is None and len(components) == 3:
+            where_clauses.append("""
+                (
+                    q.database_name > :cursor_database
+                    OR (
+                        q.database_name = :cursor_database
+                        AND (
+                            {sort_key_sql} > :cursor_sort_key
+                            OR (
+                                {sort_key_sql} = :cursor_sort_key
+                                AND q.name > :cursor_name
+                            )
+                        )
+                    )
+                )
+                """.format(sort_key_sql=sort_key_sql))
+            params["cursor_database"] = components[0]
+            params["cursor_sort_key"] = components[1]
+            params["cursor_name"] = components[2]
+        elif database is not None and len(components) == 2:
+            where_clauses.append("""
+                (
+                    {sort_key_sql} > :cursor_sort_key
+                    OR (
+                        {sort_key_sql} = :cursor_sort_key
+                        AND q.name > :cursor_name
+                    )
+                )
+                """.format(sort_key_sql=sort_key_sql))
+            params["cursor_sort_key"] = components[0]
+            params["cursor_name"] = components[1]
+
+    if q:
+        where_clauses.append("""
+            (
+                q.name LIKE :query_search
+                OR q.title LIKE :query_search
+                OR q.description LIKE :query_search
+                OR q.sql LIKE :query_search
+            )
+            """)
+        params["query_search"] = "%{}%".format(q)
+    if is_write is not None:
+        where_clauses.append("q.is_write = :query_is_write")
+        params["query_is_write"] = int(bool(is_write))
+    if is_private is not None:
+        where_clauses.append("q.is_private = :query_is_private")
+        params["query_is_private"] = int(bool(is_private))
+    if is_trusted is not None:
+        where_clauses.append("q.is_trusted = :query_is_trusted")
+        params["query_is_trusted"] = int(bool(is_trusted))
+    if source is not None:
+        where_clauses.append("q.source = :query_source")
+        params["query_source"] = source
+    if owner_id is not None:
+        where_clauses.append("q.owner_id = :query_owner_id")
+        params["query_owner_id"] = owner_id
+
+    private_select = ", allowed.is_private AS private" if include_private else ""
+    rows = list(
+        (
+            await datasette.get_internal_database().execute(
+                """
+                SELECT q.*, {sort_key_sql} AS sort_key{private_select}
+                FROM queries q
+                JOIN (
+                    {allowed_sql}
+                ) allowed
+                  ON allowed.parent = q.database_name
+                 AND allowed.child = q.name
+                WHERE {where}
+                ORDER BY {order_by}
+                LIMIT :limit
+                """.format(
+                    allowed_sql=allowed_sql,
+                    private_select=private_select,
+                    sort_key_sql=sort_key_sql,
+                    where=" AND ".join(where_clauses) or "1 = 1",
+                    order_by=order_by,
+                ),
+                params,
+            )
+        ).rows
+    )
+    has_more = len(rows) > limit
+    if has_more:
+        rows = rows[:limit]
+
+    queries = []
+    for row in rows:
+        query = query_row_to_stored_query(
+            row, private=bool(row["private"]) if include_private else None
+        )
+        assert query is not None
+        queries.append(query)
+
+    next_token = None
+    if has_more and rows:
+        last_row = rows[-1]
+        if database is None:
+            next_token = "{},{},{}".format(
+                tilde_encode(last_row["database_name"]),
+                tilde_encode(last_row["sort_key"]),
+                tilde_encode(last_row["name"]),
+            )
+        else:
+            next_token = "{},{}".format(
+                tilde_encode(last_row["sort_key"]),
+                tilde_encode(last_row["name"]),
+            )
+    return StoredQueryPage(
+        queries=queries,
+        next=next_token,
+        has_more=has_more,
+        limit=limit,
+    )
+
+
+async def ensure_query_write_permissions(
+    datasette: Any,
+    database: str,
+    sql: str,
+    *,
+    actor: dict[str, Any] | None = None,
+    params: dict[str, Any] | None = None,
+    analysis: Any = None,
+) -> Any:
+    write_actions = {
+        "insert": "insert-row",
+        "update": "update-row",
+        "delete": "delete-row",
+    }
+    db = datasette.get_database(database)
+    if analysis is None:
+        if params is None:
+            params = {name: "" for name in named_parameters(sql)}
+        try:
+            analysis = await db.analyze_sql(sql, params)
+        except sqlite3.DatabaseError as ex:
+            raise Forbidden(f"Could not analyze query: {ex}") from ex
+
+    for access in analysis.table_accesses:
+        action = write_actions.get(access.operation)
+        if action is None:
+            continue
+        if access.database != database:
+            raise Forbidden("Writable queries may not write to attached databases")
+        if not await datasette.allowed(
+            action=action,
+            resource=TableResource(database=access.database, table=access.table),
+            actor=actor,
+        ):
+            raise Forbidden(
+                f"Permission denied: need {action} on {access.database}/{access.table}"
+            )
+    return analysis

datasette/templates/_action_menu.html

@@ -1,7 +1,7 @@
 {% if action_links %}
 <div class="page-action-menu">
 <details class="actions-menu-links details-menu">
-    <summary>
+    <summary aria-haspopup="menu" aria-expanded="false">
         <div class="icon-text">
             <svg class="icon" aria-labelledby="actions-menu-links-title" role="img" style="color: #fff" xmlns="http://www.w3.org/2000/svg" width="28" height="28" viewBox="0 0 28 28" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
                 <title id="actions-menu-links-title">{{ action_title }}</title>
@@ -13,9 +13,9 @@
     </summary>
     <div class="dropdown-menu">
         <div class="hook"></div>
-        <ul>
+        <ul role="menu">
             {% for link in action_links %}
-            <li><a href="{{ link.href }}">{{ link.label }}
+            <li role="none"><a href="{{ link.href }}" role="menuitem" tabindex="-1">{{ link.label }}
             {% if link.description %}
                 <p class="dropdown-description">{{ link.description }}</p>
             {% endif %}</a>

datasette/templates/_close_open_menus.html

@@ -13,4 +13,50 @@ document.body.addEventListener('click', (ev) => {
         (details) => details.open && details != detailsClickedWithin
     ).forEach(details => details.open = false);
 });
+
+/* Sync aria-expanded and add keyboard navigation for details-menu elements */
+document.querySelectorAll('details.details-menu').forEach(function(details) {
+    var summary = details.querySelector('summary');
+    details.addEventListener('toggle', function() {
+        if (summary) {
+            summary.setAttribute('aria-expanded', details.open ? 'true' : 'false');
+        }
+        if (details.open) {
+            /* Focus first menu item when menu opens */
+            var firstItem = details.querySelector('[role="menuitem"]');
+            if (firstItem) { firstItem.focus(); }
+        }
+    });
+});
+
+document.body.addEventListener('keydown', function(ev) {
+    /* Keyboard navigation for open details-menu elements */
+    var openDetails = Array.from(document.querySelectorAll('details.details-menu[open]'));
+    if (!openDetails.length) { return; }
+
+    if (ev.key === 'Escape') {
+        openDetails.forEach(function(details) {
+            details.open = false;
+            var summary = details.querySelector('summary');
+            if (summary) { summary.focus(); }
+        });
+        return;
+    }
+
+    if (ev.key === 'ArrowDown' || ev.key === 'ArrowUp') {
+        var focused = document.activeElement;
+        openDetails.forEach(function(details) {
+            var items = Array.from(details.querySelectorAll('[role="menuitem"]'));
+            if (!items.length) { return; }
+            var idx = items.indexOf(focused);
+            if (idx === -1) { return; }
+            ev.preventDefault();
+            if (ev.key === 'ArrowDown') {
+                items[(idx + 1) % items.length].focus();
+            } else {
+                items[(idx - 1 + items.length) % items.length].focus();
+            }
+        });
+    }
+});
 </script>

datasette/templates/_execute_write_analysis_scripts.html

@@ -0,0 +1,111 @@
+<script>
+window.datasetteSqlAnalysis = (() => {
+  if (
+    window.datasetteSqlAnalysis &&
+    window.datasetteSqlAnalysis.renderAnalysis
+  ) {
+    return window.datasetteSqlAnalysis;
+  }
+
+  function appendCodeCell(row, value, emptyText) {
+    const cell = document.createElement("td");
+    if (value) {
+      const code = document.createElement("code");
+      code.textContent = value;
+      cell.appendChild(code);
+    } else if (emptyText) {
+      appendNotApplicable(cell);
+    }
+    row.appendChild(cell);
+  }
+
+  function appendNotApplicable(cell) {
+    const notApplicable = document.createElement("span");
+    notApplicable.className = "execute-write-analysis-na";
+    notApplicable.textContent = "n/a";
+    cell.appendChild(notApplicable);
+  }
+
+  function renderAnalysis(section, data) {
+    if (!section) {
+      return;
+    }
+    section.replaceChildren();
+    if (data.has_sql === false) {
+      section.hidden = true;
+      return;
+    }
+    section.hidden = false;
+
+    const heading = document.createElement("h2");
+    heading.textContent = "Query operations";
+    section.appendChild(heading);
+
+    if (data.analysis_error) {
+      const error = document.createElement("p");
+      error.className = "message-error";
+      error.textContent = data.analysis_error;
+      section.appendChild(error);
+      return;
+    }
+
+    const rows = data.analysis_rows || [];
+    if (!rows.length) {
+      const empty = document.createElement("p");
+      empty.textContent =
+        "Analysis will show each affected table and required permission.";
+      section.appendChild(empty);
+      return;
+    }
+
+    const wrapper = document.createElement("div");
+    wrapper.className = "table-wrapper";
+    const table = document.createElement("table");
+    table.className = "execute-write-analysis";
+    const thead = document.createElement("thead");
+    const headerRow = document.createElement("tr");
+    [
+      "Operation",
+      "Database",
+      "Table",
+      "Required permission",
+      "Allowed",
+    ].forEach((label) => {
+      const th = document.createElement("th");
+      th.scope = "col";
+      th.textContent = label;
+      headerRow.appendChild(th);
+    });
+    thead.appendChild(headerRow);
+    table.appendChild(thead);
+
+    const tbody = document.createElement("tbody");
+    rows.forEach((analysisRow) => {
+      const row = document.createElement("tr");
+      appendCodeCell(row, analysisRow.operation);
+      appendCodeCell(row, analysisRow.database);
+      appendCodeCell(row, analysisRow.table);
+      appendCodeCell(row, analysisRow.required_permission, "n/a");
+
+      const allowedCell = document.createElement("td");
+      if (analysisRow.allowed !== null && analysisRow.allowed !== undefined) {
+        const allowed = document.createElement("span");
+        allowed.className = analysisRow.allowed
+          ? "execute-write-analysis-allowed"
+          : "execute-write-analysis-denied";
+        allowed.textContent = analysisRow.allowed ? "yes" : "no";
+        allowedCell.appendChild(allowed);
+      } else {
+        appendNotApplicable(allowedCell);
+      }
+      row.appendChild(allowedCell);
+      tbody.appendChild(row);
+    });
+    table.appendChild(tbody);
+    wrapper.appendChild(table);
+    section.appendChild(wrapper);
+  }
+
+  return { renderAnalysis };
+})();
+</script>

datasette/templates/_execute_write_analysis_styles.html

@@ -0,0 +1,41 @@
+<style>
+.execute-write-analysis {
+    border-collapse: collapse;
+    font-size: 0.9rem;
+    margin: 0.25rem 0 1rem;
+    min-width: 44rem;
+}
+.execute-write-analysis th,
+.execute-write-analysis td {
+    border-bottom: 1px solid #d7dde5;
+    padding: 0.45rem 0.7rem;
+    text-align: left;
+    vertical-align: top;
+}
+.execute-write-analysis th {
+    background-color: #edf6fb;
+    border-top: 1px solid #d7dde5;
+    color: #39445a;
+    font-weight: 700;
+}
+.execute-write-analysis tbody tr:nth-child(even) {
+    background-color: rgba(39, 104, 144, 0.05);
+}
+.execute-write-analysis code {
+    background: transparent;
+    font-size: 0.9em;
+    white-space: nowrap;
+}
+.execute-write-analysis-allowed {
+    color: #267a3e;
+    font-weight: 700;
+}
+.execute-write-analysis-denied {
+    color: #b00020;
+    font-weight: 700;
+}
+.execute-write-analysis-na {
+    color: #687386;
+    font-style: italic;
+}
+</style>

datasette/templates/_sql_parameter_scripts.html

@@ -0,0 +1,293 @@
+<script>
+window.datasetteSqlParameters = (() => {
+  if (
+    window.datasetteSqlParameters &&
+    window.datasetteSqlParameters.setupSqlParameterRefresh
+  ) {
+    return window.datasetteSqlParameters;
+  }
+
+  function currentSql(form) {
+    if (window.editor) {
+      return window.editor.state.doc.toString();
+    }
+    const sqlInput = form.querySelector("textarea#sql-editor, input[name=sql]");
+    return sqlInput ? sqlInput.value : "";
+  }
+
+  function controlState(control) {
+    return {
+      value: control.value,
+      expanded: control.tagName.toLowerCase() === "textarea",
+    };
+  }
+
+  function syncParameterState(manager) {
+    manager.parameterState = new Map();
+    manager.section
+      .querySelectorAll("[data-parameter-control]")
+      .forEach((control) => {
+        manager.parameterState.set(control.name, controlState(control));
+      });
+  }
+
+  function createControl(parameter, id, state) {
+    const control = document.createElement(state.expanded ? "textarea" : "input");
+    control.id = id;
+    control.name = parameter;
+    control.value = state.value;
+    control.setAttribute("data-parameter-control", "");
+    if (state.expanded) {
+      control.rows = 5;
+    } else {
+      control.type = "text";
+    }
+    return control;
+  }
+
+  function replaceParameterControl(
+    manager,
+    control,
+    button,
+    expand,
+    value,
+    selectionStart
+  ) {
+    const replacement = createControl(control.name, control.id, {
+      value: value === undefined ? control.value : value,
+      expanded: expand,
+    });
+    button.textContent = expand ? "Collapse" : "Expand";
+    button.setAttribute("aria-expanded", expand ? "true" : "false");
+    control.replaceWith(replacement);
+    replacement.focus();
+    if (selectionStart !== undefined && replacement.setSelectionRange) {
+      replacement.setSelectionRange(selectionStart, selectionStart);
+    }
+    manager.parameterState.set(replacement.name, controlState(replacement));
+  }
+
+  function renderParameters(manager, parameters) {
+    syncParameterState(manager);
+    const previousState = manager.parameterState;
+    const nextState = new Map();
+    manager.section.replaceChildren();
+    if (!parameters.length) {
+      manager.parameterState = nextState;
+      return;
+    }
+
+    const heading = document.createElement("h2");
+    heading.textContent = "Parameters";
+    manager.section.appendChild(heading);
+
+    parameters.forEach((parameter, index) => {
+      const id = `qp${index + 1}`;
+      const state = previousState.get(parameter) || {
+        value: "",
+        expanded: false,
+      };
+      if (!manager.allowExpand) {
+        state.expanded = false;
+      }
+      nextState.set(parameter, state);
+
+      const row = document.createElement("p");
+      row.className = "sql-parameter-row";
+
+      const label = document.createElement("label");
+      label.htmlFor = id;
+      label.textContent = parameter;
+
+      const control = createControl(parameter, id, state);
+
+      row.append(label, control);
+      if (manager.allowExpand) {
+        const button = document.createElement("button");
+        button.type = "button";
+        button.className = "sql-parameter-toggle";
+        button.setAttribute("data-parameter-toggle", "");
+        button.setAttribute("aria-controls", id);
+        button.setAttribute("aria-expanded", state.expanded ? "true" : "false");
+        button.textContent = state.expanded ? "Collapse" : "Expand";
+        row.append(" ", button);
+      }
+      manager.section.appendChild(row);
+    });
+
+    manager.parameterState = nextState;
+  }
+
+  function bindParameterControls(manager) {
+    manager.form.addEventListener("input", (event) => {
+      const control = event.target;
+      if (!control.matches || !control.matches("[data-parameter-control]")) {
+        return;
+      }
+      manager.parameterState.set(control.name, controlState(control));
+    });
+
+    if (!manager.allowExpand) {
+      return;
+    }
+
+    manager.form.addEventListener("click", (event) => {
+      const button = event.target.closest
+        ? event.target.closest("[data-parameter-toggle]")
+        : null;
+      if (!button || !manager.form.contains(button)) {
+        return;
+      }
+      const control = document.getElementById(button.getAttribute("aria-controls"));
+      if (!control) {
+        return;
+      }
+      const expanded = control.tagName.toLowerCase() === "textarea";
+      replaceParameterControl(manager, control, button, !expanded);
+    });
+
+    manager.form.addEventListener("paste", (event) => {
+      const control = event.target;
+      if (
+        !(control instanceof HTMLInputElement) ||
+        !control.matches("[data-parameter-control]")
+      ) {
+        return;
+      }
+      const pasted = event.clipboardData ? event.clipboardData.getData("text") : "";
+      if (!/[\r\n]/.test(pasted)) {
+        return;
+      }
+      const button = document.querySelector(
+        `[data-parameter-toggle][aria-controls="${control.id}"]`
+      );
+      if (!button) {
+        return;
+      }
+      event.preventDefault();
+      const selectionStart = control.selectionStart ?? control.value.length;
+      const selectionEnd = control.selectionEnd ?? selectionStart;
+      const value =
+        control.value.slice(0, selectionStart) +
+        pasted +
+        control.value.slice(selectionEnd);
+      replaceParameterControl(
+        manager,
+        control,
+        button,
+        true,
+        value,
+        selectionStart + pasted.length
+      );
+    });
+  }
+
+  function bindEditorChanges(form, callback) {
+    const editorElement = form.querySelector(".cm-content");
+    if (editorElement) {
+      editorElement.addEventListener("input", callback);
+    }
+    if (!window.editor) {
+      const sqlInput = form.querySelector("textarea#sql-editor");
+      if (sqlInput) {
+        sqlInput.addEventListener("input", callback);
+      }
+      return;
+    }
+    if (!window.editor.datasetteSqlParameterCallbacks) {
+      const editor = window.editor;
+      const originalDispatch = editor.dispatch.bind(editor);
+      editor.datasetteSqlParameterCallbacks = [];
+      editor.dispatch = (...transactions) => {
+        const before = editor.state.doc.toString();
+        originalDispatch(...transactions);
+        if (editor.state.doc.toString() !== before) {
+          editor.datasetteSqlParameterCallbacks.forEach((listener) => listener());
+        }
+      };
+    }
+    window.editor.datasetteSqlParameterCallbacks.push(callback);
+  }
+
+  function setupSqlParameterRefresh(options) {
+    const form =
+      options.form || document.querySelector("form.sql.core[data-parameters-url]");
+    if (!form) {
+      return null;
+    }
+    const shouldRenderParameters = options.renderParameters !== false;
+    const section =
+      options.section || form.querySelector("[data-sql-parameters-section]");
+    if (shouldRenderParameters && !section) {
+      return null;
+    }
+    const manager = {
+      form,
+      section,
+      allowExpand:
+        options.allowExpand === undefined
+          ? section
+            ? section.dataset.allowExpand === "1"
+            : false
+          : options.allowExpand,
+      parameterState: new Map(),
+    };
+    if (section) {
+      bindParameterControls(manager);
+      syncParameterState(manager);
+    }
+
+    const url = options.url || form.dataset.parametersUrl;
+    let refreshTimer = null;
+    let refreshSequence = 0;
+
+    async function refreshParameters() {
+      if (!url) {
+        return;
+      }
+      const sequence = ++refreshSequence;
+      try {
+        const requestUrl = new URL(url, window.location.href);
+        requestUrl.searchParams.set("sql", currentSql(form));
+        const response = await fetch(requestUrl, {
+          headers: { accept: "application/json" },
+        });
+        const data = await response.json();
+        if (sequence !== refreshSequence) {
+          return;
+        }
+        if (!response.ok) {
+          throw new Error((data.errors || [response.statusText]).join("; "));
+        }
+        if (shouldRenderParameters) {
+          renderParameters(manager, data.parameters || []);
+        }
+        if (options.onData) {
+          options.onData(data, manager);
+        }
+      } catch (error) {
+        if (sequence !== refreshSequence) {
+          return;
+        }
+        if (options.onError) {
+          options.onError(error, manager);
+        }
+      }
+    }
+
+    function scheduleRefresh() {
+      clearTimeout(refreshTimer);
+      refreshTimer = setTimeout(refreshParameters, options.debounceMs || 350);
+    }
+
+    bindEditorChanges(form, scheduleRefresh);
+    return {
+      currentSql: () => currentSql(form),
+      refreshParameters,
+      renderParameters: (parameters) => renderParameters(manager, parameters),
+    };
+  }
+
+  return { setupSqlParameterRefresh };
+})();
+</script>

datasette/templates/_sql_parameter_styles.html

@@ -0,0 +1,58 @@
+<style>
+form.sql .sql-editor {
+    max-width: 52rem;
+}
+form.sql .sql-editor textarea#sql-editor {
+    width: 100%;
+}
+form.sql .sql-parameters-section {
+    max-width: 52rem;
+}
+form.sql .sql-parameter-row {
+    align-items: start;
+    column-gap: 0.6rem;
+    display: grid;
+    grid-template-columns: minmax(8rem, 11rem) minmax(16rem, 1fr) auto;
+    margin: 0 0 0.65rem;
+    max-width: 52rem;
+}
+form.sql .sql-parameter-row label {
+    overflow-wrap: anywhere;
+    padding-top: 0.55rem;
+    width: auto;
+}
+form.sql .sql-parameter-row input[data-parameter-control],
+form.sql .sql-parameter-row textarea[data-parameter-control] {
+    box-sizing: border-box;
+    width: 100%;
+}
+form.sql .sql-parameter-row textarea[data-parameter-control] {
+    border: 1px solid #ccc;
+    border-radius: 3px;
+    display: block;
+    font-family: Helvetica, sans-serif;
+    font-size: 1em;
+    min-height: 7rem;
+    padding: 9px 4px;
+}
+form.sql.core button.sql-parameter-toggle[type=button] {
+    font-size: 0.72rem;
+    height: 1.8rem;
+    line-height: 1;
+    margin: 0.25rem 0 0;
+    padding: 0.25rem 0.45rem;
+}
+@media (max-width: 480px) {
+    form.sql .sql-parameter-row {
+        grid-template-columns: 1fr;
+        row-gap: 0.25rem;
+    }
+    form.sql .sql-parameter-row label {
+        padding-top: 0;
+    }
+    form.sql.core button.sql-parameter-toggle[type=button] {
+        justify-self: start;
+        margin-top: 0;
+    }
+}
+</style>

datasette/templates/_sql_parameters.html

@@ -0,0 +1,9 @@
+<div id="{{ sql_parameters_section_id|default("sql-parameters-section") }}" class="sql-parameters-section" data-sql-parameters-section{% if sql_parameters_allow_expand|default(false) %} data-allow-expand="1"{% endif %}>
+    {% if parameter_names %}
+        <h2>Parameters</h2>
+        {% for parameter in parameter_names %}
+            {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %}
+            <p class="sql-parameter-row"><label for="{{ parameter_id }}">{{ parameter }}</label> <input type="text" id="{{ parameter_id }}" name="{{ parameter }}" value="{{ parameter_values.get(parameter, "") }}" data-parameter-control>{% if sql_parameters_allow_expand|default(false) %} <button type="button" class="sql-parameter-toggle" data-parameter-toggle aria-controls="{{ parameter_id }}" aria-expanded="false">Expand</button>{% endif %}</p>
+        {% endfor %}
+    {% endif %}
+</div>

datasette/templates/_table.html

@@ -1,12 +1,12 @@
 <!-- above-table-panel is a hook node for plugins to attach to . Displays even if no data available -->
 <div class="above-table-panel"> </div>
-{% if display_rows %}
+{% if display_columns %}
 <div class="table-wrapper">
     <table class="rows-and-columns">
         <thead>
             <tr>
                 {% for column in display_columns %}
-                    <th {% if column.description %}data-column-description="{{ column.description }}" {% endif %}class="col-{{ column.name|to_css_class }}" scope="col" data-column="{{ column.name }}" data-column-type="{{ column.type.lower() }}" data-column-not-null="{{ column.notnull }}" data-is-pk="{% if column.is_pk %}1{% else %}0{% endif %}">
+                    <th {% if column.description %}data-column-description="{{ column.description }}" {% endif %}class="col-{{ column.name|to_css_class }}" scope="col" data-column="{{ column.name }}" data-column-type="{{ column.type.lower() }}" data-column-not-null="{{ column.notnull }}" data-is-pk="{% if column.is_pk %}1{% else %}0{% endif %}"{% if column.is_special_link_column %} data-is-link-column="1"{% endif %}>
                         {% if not column.sortable %}
                             {{ column.name }}
                         {% else %}
@@ -31,6 +31,7 @@
         </tbody>
     </table>
 </div>
-{% else %}
+{% endif %}
+{% if not display_rows %}
     <p class="zero-results">0 records</p>
 {% endif %}

datasette/templates/base.html

@@ -20,7 +20,7 @@
 <body class="{% block body_class %}{% endblock %}">
 <div class="not-footer">
 <header class="hd"><nav>{% block nav %}{% block crumbs %}{{ crumbs.nav(request=request) }}{% endblock %}
-    {% set links = menu_links() %}{% if links or show_logout %}
+    {% set links = menu_links() %}
     <details class="nav-menu details-menu">
         <summary><svg aria-labelledby="nav-menu-svg-title" role="img"
             fill="currentColor" stroke="currentColor" xmlns="http://www.w3.org/2000/svg"
@@ -29,20 +29,18 @@
                 <path fill-rule="evenodd" d="M1 2.75A.75.75 0 011.75 2h12.5a.75.75 0 110 1.5H1.75A.75.75 0 011 2.75zm0 5A.75.75 0 011.75 7h12.5a.75.75 0 110 1.5H1.75A.75.75 0 011 7.75zM1.75 12a.75.75 0 100 1.5h12.5a.75.75 0 100-1.5H1.75z"></path>
         </svg></summary>
         <div class="nav-menu-inner">
-            {% if links %}
             <ul>
+                <li><button type="button" class="button-as-link" data-navigation-search-open aria-haspopup="dialog" aria-expanded="false" aria-keyshortcuts="/">Jump to... <kbd class="keyboard-shortcut" aria-hidden="true" title="Keyboard shortcut: press / to open Jump to">/</kbd></button></li>
                 {% for link in links %}
                 <li><a href="{{ link.href }}">{{ link.label }}</a></li>
                 {% endfor %}
             </ul>
-            {% endif %}
             {% if show_logout %}
             <form class="nav-menu-logout" action="{{ urls.logout() }}" method="post">
-                <input type="hidden" name="csrftoken" value="{{ csrftoken() }}">
                 <button class="button-as-link">Log out</button>
             </form>{% endif %}
         </div>
-    </details>{% endif %}
+    </details>
     {% if actor %}
     <div class="actor">
         <strong>{{ display_actor(actor) }}</strong>
@@ -73,6 +71,6 @@
 
 {% if select_templates %}<!-- Templates considered: {{ select_templates|join(", ") }} -->{% endif %}
 <script src="{{ urls.static('navigation-search.js') }}" defer></script>
-<navigation-search url="/-/tables"></navigation-search>
+<navigation-search url="/-/jump"></navigation-search>
 </body>
 </html>

datasette/templates/create_token.html

@@ -50,7 +50,6 @@
       </select>
     </div>
     <input type="text" name="expire_duration" style="width: 10%">
-    <input type="hidden" name="csrftoken" value="{{ csrftoken() }}">
     <input type="submit" value="Create token">
 
   <details style="margin-top: 1em" id="restrict-permissions">

datasette/templates/csrf_error.html

@@ -1,5 +1,5 @@
 {% extends "base.html" %}
-{% block title %}CSRF check failed){% endblock %}
+{% block title %}CSRF check failed{% endblock %}
 {% block content %}
 <h1>Form origin check failed</h1>
 
@@ -7,7 +7,7 @@
 
 <details><summary>Technical details</summary>
   <p>Developers: consult Datasette's <a href="https://docs.datasette.io/en/latest/internals.html#csrf-protection">CSRF protection documentation</a>.</p>
-  <p>Error code is {{ message_name }}.</p>
+  <p>Reason: {{ reason }}</p>
 </details>
 
 {% endblock %}

datasette/templates/database.html

@@ -5,6 +5,7 @@
 {% block extra_head %}
 {{- super() -}}
 {% include "_codemirror.html" %}
+{% include "_sql_parameter_styles.html" %}
 {% endblock %}
 
 {% block body_class %}db db-{{ database|to_css_class }}{% endblock %}
@@ -25,9 +26,13 @@
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
 {% if allow_execute_sql %}
-    <form class="sql core" action="{{ urls.database(database) }}/-/query" method="get">
+    <form class="sql core" action="{{ urls.database(database) }}/-/query" method="get" data-parameters-url="{{ urls.database(database) }}/-/query/parameters">
         <h3>Custom SQL query</h3>
-        <p><textarea id="sql-editor" name="sql">{% if tables %}select * from {{ tables[0].name|escape_sqlite }}{% else %}select sqlite_version(){% endif %}</textarea></p>
+        <p class="sql-editor"><textarea id="sql-editor" name="sql">{% if tables %}select * from {{ tables[0].name|escape_sqlite }}{% else %}select sqlite_version(){% endif %}</textarea></p>
+        {% set parameter_names = [] %}
+        {% set parameter_values = {} %}
+        {% set sql_parameters_allow_expand = false %}
+        {% include "_sql_parameters.html" %}
         <p>
             <button id="sql-format" type="button" hidden>Format SQL</button>
             <input type="submit" value="Run SQL">
@@ -53,10 +58,13 @@
             <li><a href="{{ urls.query(database, query.name) }}{% if query.fragment %}#{{ query.fragment }}{% endif %}" title="{{ query.description or query.sql }}">{{ query.title or query.name }}</a>{% if query.private %} 🔒{% endif %}</li>
         {% endfor %}
     </ul>
+    {% if queries_more %}
+        <p><a href="{{ urls.database(database) }}/-/queries">View {{ "{:,}".format(queries_count) }} quer{% if queries_count == 1 %}y{% else %}ies{% endif %}</a></p>
+    {% endif %}
 {% endif %}
 
 {% if tables %}
-<h2 id="tables">Tables</h2>
+<h2 id="tables">Tables <a style="font-weight: normal; font-size: 0.75em; padding-left: 0.5em;" href="{{ urls.database(database) }}/-/schema">schema</a></h2>
 {% endif %}
 
 {% for table in tables %}
@@ -87,5 +95,11 @@
 {% endif %}
 
 {% include "_codemirror_foot.html" %}
+{% include "_sql_parameter_scripts.html" %}
+<script>
+window.addEventListener("DOMContentLoaded", () => {
+  window.datasetteSqlParameters.setupSqlParameterRefresh({});
+});
+</script>
 
 {% endblock %}

datasette/templates/debug_actions.html

@@ -31,7 +31,7 @@
       <td><strong>{{ action.name }}</strong></td>
       <td>{% if action.abbr %}<code>{{ action.abbr }}</code>{% endif %}</td>
       <td>{{ action.description or "" }}</td>
-      <td><code>{{ action.resource_class }}</code></td>
+      <td>{% if action.resource_class %}<code>{{ action.resource_class }}</code>{% endif %}</td>
       <td>{% if action.takes_parent %}✓{% endif %}</td>
       <td>{% if action.takes_child %}✓{% endif %}</td>
       <td>{% if action.also_requires %}<code>{{ action.also_requires }}</code>{% endif %}</td>

datasette/templates/debug_permissions_playground.html

@@ -52,7 +52,6 @@ textarea {
 
 <div class="permission-form">
     <form action="{{ urls.path('-/permissions') }}" id="debug-post" method="post">
-        <input type="hidden" name="csrftoken" value="{{ csrftoken() }}">
         <div class="two-col">
             <div class="form-section">
                 <label>Actor</label>

datasette/templates/execute_write.html

@@ -0,0 +1,314 @@
+{% extends "base.html" %}
+
+{% block title %}Write to this database{% endblock %}
+
+{% block extra_head %}
+{{- super() -}}
+{% include "_codemirror.html" %}
+<style>
+.execute-write-template-menu {
+    margin: 0.9rem 0 0.8rem;
+    max-width: 52rem;
+}
+.execute-write-template-menu summary {
+    cursor: pointer;
+    font-weight: 600;
+    margin-bottom: 0.35rem;
+}
+.execute-write-template-controls {
+    align-items: center;
+    display: flex;
+    flex-wrap: wrap;
+    gap: 0.4rem;
+    margin: 0.4rem 0 0.7rem;
+}
+.execute-write-template-menu .execute-write-template-controls label {
+    margin-right: 0.25rem;
+    width: auto;
+}
+.execute-write-template-controls select,
+.execute-write-template-controls button[type=button] {
+    box-sizing: border-box;
+    font-size: 0.78rem;
+    height: 2rem;
+    line-height: 1.1;
+    padding: 0.35rem 0.55rem;
+}
+.execute-write-template-controls select {
+    background-color: #fff;
+    border: 1px solid #777;
+    border-radius: 0.25rem;
+    min-width: 13rem;
+}
+</style>
+{% include "_execute_write_analysis_styles.html" %}
+{% include "_sql_parameter_styles.html" %}
+{% endblock %}
+
+{% block body_class %}execute-write db-{{ database|to_css_class }}{% endblock %}
+
+{% block crumbs %}
+{{ crumbs.nav(request=request, database=database) }}
+{% endblock %}
+
+{% block content %}
+
+<h1 style="padding-left: 10px; border-left: 10px solid #{{ database_color }}">Write to this database</h1>
+
+<p>Execute SQL to insert, update or delete rows in this database.</p>
+
+{% if execution_message %}
+    <p class="{% if execution_ok %}message-info{% else %}message-error{% endif %}">{{ execution_message }}{% for link in execution_links %} <a href="{{ link.href }}">{{ link.label }}</a>{% endfor %}</p>
+{% endif %}
+
+<form class="sql core" action="{{ urls.database(database) }}/-/execute-write" method="post" data-analyze-url="{{ urls.database(database) }}/-/execute-write/analyze">
+    {% if write_template_tables %}
+        <div class="execute-write-template-menu">
+            <details>
+                <summary>Start with a template</summary>
+                <p class="execute-write-template-controls">
+                    <label for="execute-write-template-table">Table</label>
+                    <select id="execute-write-template-table">
+                        {% for table_name, columns in write_template_tables|dictsort %}
+                            <option value="{{ table_name }}">{{ table_name }}</option>
+                        {% endfor %}
+                    </select>
+                    <button type="button" data-sql-template="insert">Insert row</button>
+                    <button type="button" data-sql-template="update">Update rows</button>
+                    <button type="button" data-sql-template="delete">Delete rows</button>
+                </p>
+            </details>
+        </div>
+    {% endif %}
+
+    <p class="sql-editor"><textarea id="sql-editor" name="sql"{% if sql %} style="height: {{ sql.split("\n")|length + 2 }}em"{% endif %}>{{ sql }}</textarea></p>
+
+    {% set sql_parameters_section_id = "execute-write-parameters-section" %}
+    {% set sql_parameters_allow_expand = true %}
+    {% include "_sql_parameters.html" %}
+
+    <div id="execute-write-analysis-section">
+        <h2>Query operations</h2>
+        {% if analysis_error %}
+            <p class="message-error">{{ analysis_error }}</p>
+        {% elif analysis_rows %}
+            <div class="table-wrapper"><table class="execute-write-analysis">
+                <thead>
+                    <tr>
+                        <th scope="col">Operation</th>
+                        <th scope="col">Database</th>
+                        <th scope="col">Table</th>
+                        <th scope="col">Required permission</th>
+                        <th scope="col">Allowed</th>
+                    </tr>
+                </thead>
+                <tbody>
+                    {% for row in analysis_rows %}
+                        <tr>
+                            <td><code>{{ row.operation }}</code></td>
+                            <td><code>{{ row.database }}</code></td>
+                            <td><code>{{ row.table }}</code></td>
+                            <td>{% if row.required_permission %}<code>{{ row.required_permission }}</code>{% endif %}</td>
+                            <td>{% if row.allowed is none %}{% elif row.allowed %}<span class="execute-write-analysis-allowed">yes</span>{% else %}<span class="execute-write-analysis-denied">no</span>{% endif %}</td>
+                        </tr>
+                    {% endfor %}
+                </tbody>
+            </table></div>
+        {% else %}
+            <p>Analysis will show each affected table and required permission.</p>
+        {% endif %}
+    </div>
+
+    <p>
+        <input type="submit" value="Execute" data-execute-write-submit{% if execute_disabled %} disabled{% endif %}>
+        {% if save_query_base_url %}<a href="{{ save_query_url or save_query_base_url }}" class="save-query" data-save-query-link data-save-query-base-url="{{ save_query_base_url }}"{% if not save_query_url %} hidden{% endif %}>Save this query</a>{% endif %}
+    </p>
+</form>
+
+<script>
+const executeWriteSqlInput = document.querySelector("textarea#sql-editor");
+if (executeWriteSqlInput && !executeWriteSqlInput.value) {
+  executeWriteSqlInput.value = "\n\n\n";
+}
+</script>
+
+{% include "_codemirror_foot.html" %}
+{% include "_sql_parameter_scripts.html" %}
+{% include "_execute_write_analysis_scripts.html" %}
+
+<script>
+window.addEventListener("DOMContentLoaded", () => {
+  const form = document.querySelector("form.sql.core");
+  const analysisSection = document.querySelector("#execute-write-analysis-section");
+  const submitButton = form
+    ? form.querySelector("[data-execute-write-submit]")
+    : null;
+  const saveQueryLink = form
+    ? form.querySelector("[data-save-query-link]")
+    : null;
+
+  function updateSaveQueryLink(data) {
+    if (!saveQueryLink) {
+      return;
+    }
+    const sql = window.editor
+      ? window.editor.state.doc.toString()
+      : executeWriteSqlInput.value;
+    if (!sql.trim() || !data.ok || data.execute_disabled) {
+      saveQueryLink.hidden = true;
+      return;
+    }
+    const url = new URL(saveQueryLink.dataset.saveQueryBaseUrl, window.location.href);
+    url.searchParams.set("sql", sql);
+    saveQueryLink.href = url.pathname + url.search + url.hash;
+    saveQueryLink.hidden = false;
+  }
+
+  window.datasetteSqlParameters.setupSqlParameterRefresh({
+    form,
+    url: form.dataset.analyzeUrl,
+    allowExpand: true,
+    onData(data) {
+      window.datasetteSqlAnalysis.renderAnalysis(analysisSection, data);
+      if (submitButton) {
+        submitButton.disabled = data.execute_disabled;
+      }
+      updateSaveQueryLink(data);
+    },
+    onError(error) {
+      window.datasetteSqlAnalysis.renderAnalysis(analysisSection, {
+        analysis_error: error.message,
+        analysis_rows: [],
+      });
+      if (submitButton) {
+        submitButton.disabled = true;
+      }
+      if (saveQueryLink) {
+        saveQueryLink.hidden = true;
+      }
+    },
+  });
+});
+</script>
+
+{% if write_template_tables %}
+<script>
+window.addEventListener("DOMContentLoaded", () => {
+  const tableColumns = {{ write_template_tables|tojson(2) }};
+  const tableSelect = document.querySelector("#execute-write-template-table");
+  const templateButtons = document.querySelectorAll("[data-sql-template]");
+
+  function quoteIdentifier(identifier) {
+    return `"${identifier.replace(/"/g, '""')}"`;
+  }
+
+  function parameterNames(columns) {
+    const seen = new Set();
+    const names = {};
+    columns.forEach((column) => {
+      let base = column
+        .toLowerCase()
+        .replace(/[^a-z0-9_]+/g, "_")
+        .replace(/^_+|_+$/g, "");
+      if (!base) {
+        base = "value";
+      }
+      if (/^[0-9]/.test(base)) {
+        base = `p_${base}`;
+      }
+      let name = base;
+      let index = 2;
+      while (seen.has(name)) {
+        name = `${base}_${index}`;
+        index += 1;
+      }
+      seen.add(name);
+      names[column] = name;
+    });
+    return names;
+  }
+
+  function preferredWhereColumn(table, columns) {
+    const lowerTableId = `${table.toLowerCase()}_id`;
+    return (
+      columns.find((column) => column.toLowerCase() === "id") ||
+      columns.find((column) => column.toLowerCase() === lowerTableId) ||
+      columns[0]
+    );
+  }
+
+  function insertSql(table, columns) {
+    const names = parameterNames(columns);
+    return [
+      `insert into ${quoteIdentifier(table)} (`,
+      columns.map((column) => `  ${quoteIdentifier(column)}`).join(",\n"),
+      ")",
+      "values (",
+      columns.map((column) => `  :${names[column]}`).join(",\n"),
+      ")",
+    ].join("\n");
+  }
+
+  function updateSql(table, columns) {
+    const names = parameterNames(columns);
+    const whereColumn = preferredWhereColumn(table, columns);
+    const setColumns = columns.filter((column) => column !== whereColumn);
+    if (!setColumns.length) {
+      return [
+        `update ${quoteIdentifier(table)}`,
+        `set ${quoteIdentifier(whereColumn)} = :new_${names[whereColumn]}`,
+        `where ${quoteIdentifier(whereColumn)} = :${names[whereColumn]}`,
+      ].join("\n");
+    }
+    return [
+      `update ${quoteIdentifier(table)}`,
+      "set " +
+        setColumns
+          .map((column, index) => {
+            const indent = index ? "    " : "";
+            return `${indent}${quoteIdentifier(column)} = :${names[column]}`;
+          })
+          .join(",\n"),
+      `where ${quoteIdentifier(whereColumn)} = :${names[whereColumn]}`,
+    ].join("\n");
+  }
+
+  function deleteSql(table, columns) {
+    const names = parameterNames(columns);
+    const whereColumn = preferredWhereColumn(table, columns);
+    return [
+      `delete from ${quoteIdentifier(table)}`,
+      `where ${quoteIdentifier(whereColumn)} = :${names[whereColumn]}`,
+    ].join("\n");
+  }
+
+  function templateSql(operation, table, columns) {
+    if (operation === "insert") {
+      return insertSql(table, columns);
+    }
+    if (operation === "update") {
+      return updateSql(table, columns);
+    }
+    return deleteSql(table, columns);
+  }
+
+  templateButtons.forEach((button) => {
+    button.addEventListener("click", () => {
+      const table = tableSelect.value;
+      const columns = tableColumns[table] || [];
+      if (!columns.length) {
+        return;
+      }
+      const url = new URL(window.location.href);
+      url.searchParams.set(
+        "sql",
+        templateSql(button.dataset.sqlTemplate, table, columns)
+      );
+      window.location.href = url.toString();
+    });
+  });
+});
+</script>
+{% endif %}
+
+{% endblock %}

datasette/templates/logout.html

@@ -10,7 +10,6 @@
 
 <form class="core" action="{{ urls.logout() }}" method="post">
     <div>
-        <input type="hidden" name="csrftoken" value="{{ csrftoken() }}">
         <input type="submit" value="Log out">
     </div>
 </form>

datasette/templates/messages_debug.html

@@ -19,7 +19,6 @@
                 <option>all</option>
             </select>
         </div>
-        <input type="hidden" name="csrftoken" value="{{ csrftoken() }}">
         <input type="submit" value="Add message">
     </div>
 </form>

datasette/templates/query.html

@@ -14,9 +14,10 @@
 </style>
 {% endif %}
 {% include "_codemirror.html" %}
+{% include "_sql_parameter_styles.html" %}
 {% endblock %}
 
-{% block body_class %}query db-{{ database|to_css_class }}{% if canned_query %} query-{{ canned_query|to_css_class }}{% endif %}{% endblock %}
+{% block body_class %}query db-{{ database|to_css_class }}{% if stored_query %} query-{{ stored_query|to_css_class }}{% endif %}{% endblock %}
 
 {% block crumbs %}
 {{ crumbs.nav(request=request, database=database) }}
@@ -24,19 +25,19 @@
 
 {% block content %}
 
-{% if canned_query_write and db_is_immutable %}
+{% if stored_query_write and db_is_immutable %}
     <p class="message-error">This query cannot be executed because the database is immutable.</p>
 {% endif %}
 
-<h1 style="padding-left: 10px; border-left: 10px solid #{{ database_color }}">{{ metadata.title or database }}{% if canned_query and not metadata.title %}: {{ canned_query }}{% endif %}{% if private %} 🔒{% endif %}</h1>
+<h1 style="padding-left: 10px; border-left: 10px solid #{{ database_color }}">{{ metadata.title or database }}{% if stored_query and not metadata.title %}: {{ stored_query }}{% endif %}{% if private %} 🔒{% endif %}</h1>
 {% set action_links, action_title = query_actions(), "Query actions" %}
 {% include "_action_menu.html" %}
 
-{% if canned_query %}{{ top_canned_query() }}{% else %}{{ top_query() }}{% endif %}
+{% if stored_query %}{{ top_stored_query() }}{% else %}{{ top_query() }}{% endif %}
 
 {% block description_source_license %}{% include "_description_source_license.html" %}{% endblock %}
 
-<form class="sql core" action="{{ urls.database(database) }}{% if canned_query %}/{{ canned_query }}{% endif %}" method="{% if canned_query_write %}post{% else %}get{% endif %}">
+<form class="sql core" action="{{ urls.database(database) }}{% if stored_query %}/{{ stored_query }}{% endif %}" method="{% if stored_query_write %}post{% else %}get{% endif %}" data-parameters-url="{{ urls.database(database) }}/-/query/parameters">
     <h3>Custom SQL query{% if display_rows %} returning {% if truncated %}more than {% endif %}{{ "{:,}".format(display_rows|length) }} row{% if display_rows|length == 1 %}{% else %}s{% endif %}{% endif %}{% if not query_error %}
         <span class="show-hide-sql">(<a href="{{ show_hide_link }}">{{ show_hide_text }}</a>)</span>
     {% endif %}</h3>
@@ -45,30 +46,28 @@
     {% endif %}
     {% if not hide_sql %}
         {% if editable and allow_execute_sql %}
-            <p><textarea id="sql-editor" name="sql"{% if query and query.sql %} style="height: {{ query.sql.split("\n")|length + 2 }}em"{% endif %}
-            >{% if query and query.sql %}{{ query.sql }}{% else %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}</textarea></p>
+            <p class="sql-editor"><textarea id="sql-editor" name="sql"{% if query and query.sql %} style="height: {{ query.sql.split("\n")|length + 2 }}em"{% endif %}
+            >{% if query and query.sql %}{{ query.sql }}{% elif tables %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}</textarea></p>
         {% else %}
             <pre id="sql-query">{% if query %}{{ query.sql }}{% endif %}</pre>
         {% endif %}
     {% else %}
-        {% if not canned_query %}
+        {% if not stored_query %}
             <input type="hidden" name="sql"
-                value="{% if query and query.sql %}{{ query.sql }}{% else %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}"
+                value="{% if query and query.sql %}{{ query.sql }}{% elif tables %}select * from {{ tables[0].name|escape_sqlite }}{% endif %}"
             >
         {% endif %}
     {% endif %}
-    {% if named_parameter_values %}
-        <h3>Query parameters</h3>
-        {% for name, value in named_parameter_values.items() %}
-            <p><label for="qp{{ loop.index }}">{{ name }}</label> <input type="text" id="qp{{ loop.index }}" name="{{ name }}" value="{{ value }}"></p>
-        {% endfor %}
-    {% endif %}
+    {% set parameter_names = named_parameter_values.keys()|list %}
+    {% set parameter_values = named_parameter_values %}
+    {% set sql_parameters_allow_expand = false %}
+    {% include "_sql_parameters.html" %}
     <p>
         {% if not hide_sql %}<button id="sql-format" type="button" hidden>Format SQL</button>{% endif %}
-        {% if canned_query_write %}<input type="hidden" name="csrftoken" value="{{ csrftoken() }}">{% endif %}
-        <input type="submit" value="Run SQL"{% if canned_query_write and db_is_immutable %} disabled{% endif %}>
+        <input type="submit" value="Run SQL"{% if stored_query_write and db_is_immutable %} disabled{% endif %}>
         {{ show_hide_hidden }}
-        {% if canned_query and edit_sql_url %}<a href="{{ edit_sql_url }}" class="canned-query-edit-sql">Edit SQL</a>{% endif %}
+        {% if save_query_url %}<a href="{{ save_query_url }}" class="save-query">Save this query</a>{% endif %}
+        {% if stored_query and edit_sql_url %}<a href="{{ edit_sql_url }}" class="stored-query-edit-sql">Edit SQL</a>{% endif %}
     </p>
 </form>
 
@@ -91,11 +90,17 @@
     </tbody>
 </table></div>
 {% else %}
-    {% if not canned_query_write and not error %}
+    {% if not stored_query_write and not error %}
         <p class="zero-results">0 results</p>
     {% endif %}
 {% endif %}
 
 {% include "_codemirror_foot.html" %}
+{% include "_sql_parameter_scripts.html" %}
+<script>
+window.addEventListener("DOMContentLoaded", () => {
+  window.datasetteSqlParameters.setupSqlParameterRefresh({});
+});
+</script>
 
 {% endblock %}

datasette/templates/query_create.html

@@ -0,0 +1,302 @@
+{% extends "base.html" %}
+
+{% block title %}Create query{% endblock %}
+
+{% block extra_head %}
+{{- super() -}}
+{% include "_codemirror.html" %}
+{% include "_execute_write_analysis_styles.html" %}
+<style>
+.query-create-page {
+    max-width: 64rem;
+}
+.query-create-form {
+    --query-create-label-width: clamp(7rem, 18vw, 10rem);
+    --query-create-column-gap: 0.8rem;
+    --query-create-control-width: minmax(16rem, 1fr);
+}
+.query-create-fields {
+    margin: 0 0 0.85rem;
+    max-width: 52rem;
+}
+.query-create-field {
+    align-items: start;
+    column-gap: var(--query-create-column-gap);
+    display: grid;
+    grid-template-columns: var(--query-create-label-width) var(--query-create-control-width);
+    margin: 0 0 0.65rem;
+}
+.query-create-field label {
+    padding-top: 0.55rem;
+    width: auto;
+}
+.query-create-field input[type=text],
+.query-create-field textarea {
+    box-sizing: border-box;
+    width: 100%;
+}
+form.sql .query-create-field textarea {
+    width: 100%;
+}
+.query-create-url-control {
+    align-items: center;
+    box-sizing: border-box;
+    display: grid;
+    gap: 0.35rem;
+    grid-template-columns: max-content minmax(12rem, 1fr);
+    width: 100%;
+}
+.query-create-url-prefix {
+    color: #4f5b6d;
+    font-family: var(--font-monospace, monospace);
+    white-space: nowrap;
+}
+.query-create-url-control input[type=text] {
+    border: 1px solid #ccc;
+    border-radius: 3px;
+}
+.query-create-field textarea {
+    border: 1px solid #ccc;
+    border-radius: 3px;
+    display: block;
+    font-family: Helvetica, sans-serif;
+    font-size: 1em;
+    min-height: 5rem;
+    padding: 9px 4px;
+    resize: vertical;
+}
+form.sql .query-create-sql {
+    column-gap: var(--query-create-column-gap);
+    display: grid;
+    grid-template-columns: var(--query-create-label-width) var(--query-create-control-width);
+    margin: 0.9rem 0 0.75rem;
+    max-width: 52rem;
+}
+.query-create-sql .cm-editor,
+form.sql .query-create-sql textarea#sql-editor {
+    grid-column: 2;
+    width: 100%;
+}
+.query-create-options {
+    align-items: center;
+    display: flex;
+    flex-wrap: wrap;
+    gap: 0.8rem 1.4rem;
+    margin: 0 0 0.9rem calc(var(--query-create-label-width) + var(--query-create-column-gap));
+    max-width: calc(52rem - var(--query-create-label-width) - var(--query-create-column-gap));
+}
+.query-create-options label {
+    align-items: center;
+    display: inline-flex;
+    gap: 0.35rem;
+    width: auto;
+}
+.query-create-options input[type=checkbox] {
+    margin: 0;
+}
+.query-create-option-note,
+.query-create-analysis-note {
+    color: #4f5b6d;
+    flex-basis: 100%;
+    font-size: 0.82rem;
+}
+.query-create-option-note {
+    margin: -0.45rem 0 0;
+}
+.query-create-analysis-note {
+    margin: 0;
+}
+.query-create-action {
+    margin: 0.35rem 0 1rem;
+}
+.query-create-analysis {
+    margin-top: 0.8rem;
+}
+.query-create-submit {
+    margin-left: calc(var(--query-create-label-width) + var(--query-create-column-gap));
+    margin-bottom: 0.9rem;
+    margin-top: 1rem;
+}
+@media (max-width: 560px) {
+    .query-create-form {
+        --query-create-label-width: 1fr;
+        --query-create-column-gap: 0;
+    }
+    .query-create-field {
+        grid-template-columns: 1fr;
+        row-gap: 0.25rem;
+    }
+    .query-create-field label {
+        padding-top: 0;
+    }
+    form.sql .query-create-sql {
+        grid-template-columns: 1fr;
+    }
+    .query-create-sql .cm-editor,
+    form.sql .query-create-sql textarea#sql-editor {
+        grid-column: 1;
+    }
+    .query-create-options,
+    .query-create-submit {
+        margin-left: 0;
+    }
+}
+</style>
+{% endblock %}
+
+{% block body_class %}query-create db-{{ database|to_css_class }}{% endblock %}
+
+{% block crumbs %}
+{{ crumbs.nav(request=request, database=database) }}
+{% endblock %}
+
+{% block content %}
+
+<div class="query-create-page">
+
+<h1 style="padding-left: 10px; border-left: 10px solid #{{ database_color }}">Create query</h1>
+
+<form class="sql core query-create-form" action="{{ urls.database(database) }}/-/queries/store" method="post" data-analyze-url="{{ urls.database(database) }}/-/queries/analyze">
+    <div class="query-create-fields">
+        <p class="query-create-field"><label for="query-title">Title</label> <input id="query-title" name="title" type="text" value="{{ title or "" }}"></p>
+        <p class="query-create-field"><label for="query-url-slug">URL</label> <span class="query-create-url-control"><span class="query-create-url-prefix">{{ urls.database(database) }}/</span><input id="query-url-slug" name="name" type="text" value="{{ name or "" }}"></span></p>
+        <p class="query-create-field"><label for="query-description">Description</label> <textarea id="query-description" name="description" rows="3">{{ description or "" }}</textarea></p>
+    </div>
+
+    <p class="query-create-sql sql-editor"><textarea id="sql-editor" name="sql"{% if sql %} style="height: {{ sql.split("\n")|length + 2 }}em"{% endif %}>{{ sql }}</textarea></p>
+
+    <p class="query-create-options">
+        <span class="query-create-analysis-note" data-query-create-analysis-note aria-live="polite">{% if analysis_error %}This query cannot be saved until the SQL is valid.{% elif not has_sql %}Enter SQL to analyze this query.{% elif analysis_is_write %}This query updates data in the database.{% else %}This is a read-only query.{% endif %}</span>
+        <input type="hidden" name="is_private" value="0">
+        <label><input type="checkbox" name="is_private" value="1"{% if is_private %} checked{% endif %}> Private</label>
+        <span class="query-create-option-note">Queries marked private can only be seen by you, their creator.</span>
+    </p>
+    {% if sql and analysis_is_write %}
+        <p class="query-create-action"><a href="{{ urls.database(database) }}/-/execute-write?{{ {'sql': sql}|urlencode|safe }}">Execute write SQL</a></p>
+    {% endif %}
+
+    <p class="query-create-submit"><input type="submit" value="Save query" data-query-create-submit{% if save_disabled %} disabled{% endif %}></p>
+
+    <div class="query-create-analysis" id="query-create-analysis-section"{% if not has_sql %} hidden{% endif %}>
+        {% if has_sql %}
+            <h2>Query operations</h2>
+            {% if analysis_error %}
+                <p class="message-error">{{ analysis_error }}</p>
+            {% elif analysis_rows %}
+                <div class="table-wrapper"><table class="execute-write-analysis">
+                    <thead>
+                        <tr>
+                            <th scope="col">Operation</th>
+                            <th scope="col">Database</th>
+                            <th scope="col">Table</th>
+                            <th scope="col">Required permission</th>
+                            <th scope="col">Allowed</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for row in analysis_rows %}
+                            <tr>
+                                <td><code>{{ row.operation }}</code></td>
+                                <td><code>{{ row.database }}</code></td>
+                                <td><code>{{ row.table }}</code></td>
+                                <td>{% if row.required_permission %}<code>{{ row.required_permission }}</code>{% else %}<span class="execute-write-analysis-na">n/a</span>{% endif %}</td>
+                                <td>{% if row.allowed is none %}<span class="execute-write-analysis-na">n/a</span>{% elif row.allowed %}<span class="execute-write-analysis-allowed">yes</span>{% else %}<span class="execute-write-analysis-denied">no</span>{% endif %}</td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table></div>
+            {% else %}
+                <p>Analysis will show each affected table and required permission.</p>
+            {% endif %}
+        {% endif %}
+    </div>
+</form>
+
+</div>
+
+{% include "_codemirror_foot.html" %}
+{% include "_sql_parameter_scripts.html" %}
+{% include "_execute_write_analysis_scripts.html" %}
+
+<script>
+window.addEventListener("DOMContentLoaded", () => {
+  const titleInput = document.querySelector("#query-title");
+  const urlInput = document.querySelector("#query-url-slug");
+  let urlEdited = Boolean(urlInput && urlInput.value);
+
+  function slugify(value) {
+    return value
+      .normalize("NFKD")
+      .replace(/[\u0300-\u036f]/g, "")
+      .toLowerCase()
+      .trim()
+      .replace(/[^a-z0-9_-]+/g, "-")
+      .replace(/-+/g, "-")
+      .replace(/^-|-$/g, "");
+  }
+
+  if (titleInput && urlInput) {
+    titleInput.addEventListener("input", () => {
+      if (!urlEdited) {
+        urlInput.value = slugify(titleInput.value);
+      }
+    });
+    urlInput.addEventListener("input", () => {
+      urlEdited = true;
+    });
+  }
+});
+</script>
+
+<script>
+window.addEventListener("DOMContentLoaded", () => {
+  const form = document.querySelector("form.sql.core");
+  const analysisSection = document.querySelector("#query-create-analysis-section");
+  const submitButton = form
+    ? form.querySelector("[data-query-create-submit]")
+    : null;
+  const analysisNote = form
+    ? form.querySelector("[data-query-create-analysis-note]")
+    : null;
+
+  function updateAnalysisNote(data) {
+    if (!analysisNote) {
+      return;
+    }
+    if (data.analysis_error) {
+      analysisNote.textContent = "This query cannot be saved until the SQL is valid.";
+    } else if (data.has_sql === false) {
+      analysisNote.textContent = "Enter SQL to analyze this query.";
+    } else if (data.analysis_is_write) {
+      analysisNote.textContent = "This query updates data in the database.";
+    } else {
+      analysisNote.textContent = "This is a read-only query.";
+    }
+  }
+
+  window.datasetteSqlParameters.setupSqlParameterRefresh({
+    form,
+    url: form.dataset.analyzeUrl,
+    renderParameters: false,
+    onData(data) {
+      window.datasetteSqlAnalysis.renderAnalysis(analysisSection, data);
+      if (submitButton) {
+        submitButton.disabled = data.save_disabled;
+      }
+      updateAnalysisNote(data);
+    },
+    onError(error) {
+      window.datasetteSqlAnalysis.renderAnalysis(analysisSection, {
+        analysis_error: error.message,
+        analysis_rows: [],
+      });
+      if (submitButton) {
+        submitButton.disabled = true;
+      }
+      updateAnalysisNote({ analysis_error: error.message });
+    },
+  });
+});
+</script>
+
+{% endblock %}

datasette/templates/query_list.html

@@ -0,0 +1,281 @@
+{% extends "base.html" %}
+
+{% block title %}{% if database %}{{ database }}: {% endif %}queries{% endblock %}
+
+{% block extra_head %}
+{{- super() -}}
+<style>
+.query-list-page {
+    max-width: 64rem;
+}
+.query-list-filters {
+    margin: 0.5rem 0 0.75rem;
+}
+.query-list-search {
+    align-items: center;
+    display: flex;
+    flex-wrap: wrap;
+    gap: 0.45rem;
+    margin: 0 0 0.75rem;
+}
+.query-list-search label {
+    width: auto;
+}
+.query-list-search input[type=search] {
+    box-sizing: border-box;
+    flex: 1 1 18rem;
+    max-width: 24rem;
+}
+.query-list-search button[type=submit] {
+    font-size: 0.78rem;
+    height: 2rem;
+    line-height: 1.1;
+    padding: 0.35rem 0.65rem;
+}
+.query-list-facets {
+    align-items: flex-start;
+    display: flex;
+    flex-wrap: wrap;
+    gap: 1rem 1.6rem;
+    margin: 0 0 1rem;
+}
+.query-list-facet {
+    margin: 0;
+}
+.query-list-facet h2 {
+    font-size: 0.9rem;
+    line-height: 1.2;
+    margin: 0 0 0.35rem;
+}
+.query-list-facet ul {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 0.35rem;
+    margin: 0;
+    padding: 0;
+    list-style: none;
+}
+.query-list-facet-link,
+.query-list-facet-link:link,
+.query-list-facet-link:visited,
+.query-list-facet-link:hover,
+.query-list-facet-link:focus,
+.query-list-facet-link:active {
+    align-items: center;
+    border: 1px solid #c8d1dc;
+    border-radius: 0.25rem;
+    color: #39445a;
+    display: inline-flex;
+    font-size: 0.82rem;
+    gap: 0.4rem;
+    line-height: 1.1;
+    padding: 0.35rem 0.55rem;
+    text-decoration: none;
+}
+.query-list-facet-link:hover {
+    border-color: #7ca5c8;
+    color: #1f5d85;
+}
+.query-list-facet-link-active {
+    background-color: #edf6fb;
+    border-color: #6d9fc0;
+    font-weight: 700;
+}
+.query-list-facet-disabled {
+    color: #7b8794;
+    cursor: default;
+}
+.query-list-facet-count {
+    color: #4f5b6d;
+    font-variant-numeric: tabular-nums;
+}
+.query-list-results {
+    border-collapse: collapse;
+    font-size: 0.9rem;
+    margin: 0.25rem 0 1rem;
+    min-width: 42rem;
+    width: 100%;
+}
+.query-list-results th,
+.query-list-results td {
+    border-bottom: 1px solid #d7dde5;
+    padding: 0.45rem 0.7rem;
+    text-align: left;
+    vertical-align: top;
+}
+.query-list-results th {
+    background-color: #edf6fb;
+    border-top: 1px solid #d7dde5;
+    color: #39445a;
+    font-weight: 700;
+}
+.query-list-results tbody tr:nth-child(even) {
+    background-color: rgba(39, 104, 144, 0.05);
+}
+.query-list-results a.query-list-title {
+    font-weight: 700;
+}
+.query-list-description {
+    color: #4f5b6d;
+    font-size: 0.78rem;
+    margin: 0.15rem 0 0;
+}
+.query-list-owner {
+    color: #39445a;
+    font-family: var(--font-monospace, monospace);
+    white-space: nowrap;
+}
+.query-list-flags {
+    display: flex;
+    flex-wrap: wrap;
+    gap: 0.3rem;
+}
+.query-list-pill {
+    background-color: #eef1f5;
+    border: 1px solid #d7dde5;
+    border-radius: 0.25rem;
+    color: #39445a;
+    display: inline-block;
+    font-size: 0.75rem;
+    font-weight: 700;
+    line-height: 1;
+    padding: 0.25rem 0.4rem;
+    white-space: nowrap;
+}
+.query-list-pill-write {
+    background-color: #fff4db;
+    border-color: #e2b64e;
+}
+.query-list-pill-public {
+    background-color: #e7f5ec;
+    border-color: #9ecfab;
+    color: #267a3e;
+}
+.query-list-pill-private {
+    background-color: #f7edf0;
+    border-color: #dbb8c1;
+}
+.query-list-pill-trusted {
+    background-color: #e7f5ec;
+    border-color: #9ecfab;
+    color: #267a3e;
+}
+.query-list-empty {
+    color: #6b7280;
+}
+.query-list-footnotes {
+    border-top: 1px solid #d7dde5;
+    color: #4f5b6d;
+    font-size: 0.82rem;
+    margin: 0.35rem 0 1rem;
+    padding-top: 0.55rem;
+}
+.query-list-footnotes p {
+    margin: 0.25rem 0;
+}
+.query-list-footnotes .query-list-pill {
+    margin-right: 0.35rem;
+}
+.query-list-pagination a {
+    border: 1px solid #007bff;
+    border-radius: 0.25rem;
+    display: inline-block;
+    padding: 0.45rem 0.7rem;
+}
+.query-list-pagination-bottom {
+    margin-top: 0.75rem;
+}
+@media (max-width: 700px) {
+    .query-list-search input[type=search] {
+        max-width: none;
+    }
+}
+</style>
+{% endblock %}
+
+{% block body_class %}query-list{% if database %} db-{{ database|to_css_class }}{% endif %}{% endblock %}
+
+{% block crumbs %}
+{{ crumbs.nav(request=request, database=database) }}
+{% endblock %}
+
+{% block content %}
+
+<div class="query-list-page">
+
+<h1 style="padding-left: 10px; border-left: 10px solid #{% if database_color %}{{ database_color }}{% else %}666{% endif %}">Queries</h1>
+
+<form class="query-list-filters core" action="{{ query_list_path }}" method="get">
+    <p class="query-list-search">
+        <label for="query-search">Search</label>
+        <input id="query-search" type="search" name="q" value="{{ filters.q }}">
+        {% if filters.is_write %}<input type="hidden" name="is_write" value="{{ filters.is_write }}">{% endif %}
+        {% if filters.is_private %}<input type="hidden" name="is_private" value="{{ filters.is_private }}">{% endif %}
+        {% if filters.source %}<input type="hidden" name="source" value="{{ filters.source }}">{% endif %}
+        {% if filters.owner_id %}<input type="hidden" name="owner_id" value="{{ filters.owner_id }}">{% endif %}
+        <button type="submit">Search</button>
+    </p>
+</form>
+
+<nav class="query-list-facets" aria-label="Query filters">
+    {% for facet in facets %}
+        <section class="query-list-facet">
+            <h2>{{ facet.title }}</h2>
+            <ul>
+                {% for item in facet["items"] %}
+                    <li>{% if item.href %}<a class="query-list-facet-link{% if item.active %} query-list-facet-link-active{% endif %}" href="{{ item.href }}"{% if item.active %} aria-current="true"{% endif %}>{% else %}<span class="query-list-facet-link query-list-facet-disabled">{% endif %}<span>{{ item.label }}</span><span class="query-list-facet-count">{{ item.count }}</span>{% if item.href %}</a>{% else %}</span>{% endif %}</li>
+                {% endfor %}
+            </ul>
+        </section>
+    {% endfor %}
+</nav>
+
+{% if queries %}
+    <div class="table-wrapper"><table class="query-list-results">
+        <thead>
+            <tr>
+                {% if show_database %}<th scope="col">Database</th>{% endif %}
+                <th scope="col">Query</th>
+                <th scope="col">Owner</th>
+                <th scope="col">Flags</th>
+            </tr>
+        </thead>
+        <tbody>
+            {% for query in queries %}
+                <tr>
+                    {% if show_database %}
+                        <td><a class="query-list-database" href="{{ urls.database(query.database) }}">{{ query.database }}</a></td>
+                    {% endif %}
+                    <td>
+                        <a class="query-list-title" href="{{ urls.query(query.database, query.name) }}{% if query.fragment %}#{{ query.fragment }}{% endif %}" title="{{ query.description or query.sql }}">{{ query.title or query.name }}</a>{% if query.private %} 🔒{% endif %}
+                        {% if query.description %}<p class="query-list-description">{{ query.description }}</p>{% endif %}
+                    </td>
+                    <td class="query-list-owner">{% if query.owner_id is not none %}{{ query.owner_id }}{% else %}<span class="query-list-empty">-</span>{% endif %}</td>
+                    <td>
+                        <span class="query-list-flags">
+                            {% if query.is_write %}<span class="query-list-pill query-list-pill-write">Writable</span>{% else %}<span class="query-list-pill">Read-only</span>{% endif %}
+                            {% if query.is_private %}<span class="query-list-pill query-list-pill-private">Private</span>{% endif %}
+                            {% if query.is_trusted %}<span class="query-list-pill query-list-pill-trusted">Trusted</span>{% endif %}
+                        </span>
+                    </td>
+                </tr>
+            {% endfor %}
+        </tbody>
+    </table></div>
+    {% if show_private_note or show_trusted_note %}
+        <div class="query-list-footnotes">
+            {% if show_private_note %}<p><span class="query-list-pill query-list-pill-private">Private</span>Only the owning actor can view this query.</p>{% endif %}
+            {% if show_trusted_note %}<p><span class="query-list-pill query-list-pill-trusted">Trusted</span>Execution skips the usual SQL and write permission checks after view-query allows access.</p>{% endif %}
+        </div>
+    {% endif %}
+{% else %}
+    <p>No queries found.</p>
+{% endif %}
+
+{% if next_url %}
+    <nav class="query-list-pagination query-list-pagination-bottom" aria-label="Query pagination"><a href="{{ next_url }}">Next page</a></nav>
+{% endif %}
+
+</div>
+
+{% endblock %}

datasette/templates/schema.html

@@ -0,0 +1,41 @@
+{% extends "base.html" %}
+
+{% block title %}{% if is_instance %}Schema for all databases{% elif table_name %}Schema for {{ schemas[0].database }}.{{ table_name }}{% else %}Schema for {{ schemas[0].database }}{% endif %}{% endblock %}
+
+{% block body_class %}schema{% endblock %}
+
+{% block crumbs %}
+{% if is_instance %}
+{{ crumbs.nav(request=request) }}
+{% elif table_name %}
+{{ crumbs.nav(request=request, database=schemas[0].database, table=table_name) }}
+{% else %}
+{{ crumbs.nav(request=request, database=schemas[0].database) }}
+{% endif %}
+{% endblock %}
+
+{% block content %}
+<div class="page-header">
+    <h1>{% if is_instance %}Schema for all databases{% elif table_name %}Schema for {{ table_name }}{% else %}Schema for {{ schemas[0].database }}{% endif %}</h1>
+</div>
+
+{% for item in schemas %}
+    {% if is_instance %}
+        <h2>{{ item.database }}</h2>
+    {% endif %}
+
+    {% if item.schema %}
+        <pre style="background-color: #f5f5f5; padding: 1em; overflow-x: auto; border: 1px solid #ddd; border-radius: 4px;"><code>{{ item.schema }}</code></pre>
+    {% else %}
+        <p><em>No schema available for this database.</em></p>
+    {% endif %}
+
+    {% if not loop.last %}
+        <hr style="margin: 2em 0;">
+    {% endif %}
+{% endfor %}
+
+{% if not schemas %}
+    <p><em>No databases with viewable schemas found.</em></p>
+{% endif %}
+{% endblock %}

datasette/templates/table.html

@@ -4,7 +4,9 @@
 
 {% block extra_head %}
 {{- super() -}}
+<script src="{{ urls.static('column-chooser.js') }}" defer></script>
 <script src="{{ urls.static('table.js') }}" defer></script>
+<script src="{{ urls.static('mobile-column-actions.js') }}" defer></script>
 <script>DATASETTE_ALLOW_FACET = {{ datasette_allow_facet }};</script>
 <style>
 @media only screen and (max-width: 576px) {
@@ -136,6 +138,26 @@
     {% include "_facet_results.html" %}
 {% endif %}
 
+{% if all_columns %}
+<column-chooser></column-chooser>
+<button class="choose-columns-mobile small-screen-only" onclick="openColumnChooser()">Choose columns</button>
+<button type="button" class="column-actions-mobile small-screen-only">
+    <svg aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+        <circle cx="12" cy="12" r="3"></circle>
+        <path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1-2-2 2 2 0 0 1 2-2h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 2-2 2 2 0 0 1 2 2v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-.09a1.65 1.65 0 0 0-1.51 1z"></path>
+    </svg>
+    <span>Column actions</span>
+</button>
+<script>
+window._columnChooserData = {{ {"allColumns": all_columns, "selectedColumns": display_columns|map(attribute='name')|list, "primaryKeys": primary_keys}|tojson }};
+</script>
+{% endif %}
+{% if set_column_type_ui %}
+<script>
+window._setColumnTypeData = {{ set_column_type_ui|tojson }};
+</script>
+{% endif %}
+
 {% include custom_table_templates %}
 
 {% if next_url %}

datasette/tokens.py

@@ -0,0 +1,193 @@
+"""
+Token handler system for Datasette.
+
+Provides a base class for token handlers and the default signed token handler.
+Plugins can implement register_token_handler to provide custom token backends
+(e.g. database-backed tokens that can be revoked and audited).
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import time
+from typing import TYPE_CHECKING, Optional
+
+import itsdangerous
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+
+@dataclasses.dataclass
+class TokenRestrictions:
+    """
+    Restrictions to apply to a token, limiting which actions it can perform.
+
+    Use the builder methods to construct restrictions::
+
+        restrictions = (TokenRestrictions()
+            .allow_all("view-instance")
+            .allow_database("mydb", "create-table")
+            .allow_resource("mydb", "mytable", "insert-row"))
+    """
+
+    all: list[str] = dataclasses.field(default_factory=list)
+    database: dict[str, list[str]] = dataclasses.field(default_factory=dict)
+    resource: dict[str, dict[str, list[str]]] = dataclasses.field(default_factory=dict)
+
+    def allow_all(self, action: str) -> "TokenRestrictions":
+        """Allow an action across all databases and resources."""
+        self.all.append(action)
+        return self
+
+    def allow_database(self, database: str, action: str) -> "TokenRestrictions":
+        """Allow an action on a specific database."""
+        self.database.setdefault(database, []).append(action)
+        return self
+
+    def allow_resource(
+        self, database: str, resource: str, action: str
+    ) -> "TokenRestrictions":
+        """Allow an action on a specific resource within a database."""
+        self.resource.setdefault(database, {}).setdefault(resource, []).append(action)
+        return self
+
+    def abbreviated(self, datasette: "Datasette") -> Optional[dict]:
+        """
+        Return the abbreviated ``_r`` dictionary shape for this set of
+        restrictions, using action abbreviations registered with ``datasette``.
+        Returns ``None`` if no restrictions are set.
+        """
+        if not (self.all or self.database or self.resource):
+            return None
+
+        def abbreviate_action(action):
+            action_obj = datasette.actions.get(action)
+            if not action_obj:
+                return action
+            return action_obj.abbr or action
+
+        result: dict = {}
+        if self.all:
+            result["a"] = [abbreviate_action(a) for a in self.all]
+        if self.database:
+            result["d"] = {
+                database: [abbreviate_action(a) for a in actions]
+                for database, actions in self.database.items()
+            }
+        if self.resource:
+            result["r"] = {}
+            for database, resources in self.resource.items():
+                for resource, actions in resources.items():
+                    result["r"].setdefault(database, {})[resource] = [
+                        abbreviate_action(a) for a in actions
+                    ]
+        return result
+
+
+class TokenHandler:
+    """
+    Base class for token handlers.
+
+    Subclass this and implement create_token() and verify_token() to provide
+    a custom token backend. Return an instance from the register_token_handler hook.
+    """
+
+    name: str = ""
+
+    async def create_token(
+        self,
+        datasette: "Datasette",
+        actor_id: str,
+        *,
+        expires_after: Optional[int] = None,
+        restrictions: Optional[TokenRestrictions] = None,
+    ) -> str:
+        """Create and return a token string for the given actor."""
+        raise NotImplementedError
+
+    async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]:
+        """
+        Verify a token and return an actor dict, or None if this handler
+        does not recognize the token.
+        """
+        raise NotImplementedError
+
+
+class SignedTokenHandler(TokenHandler):
+    """
+    Default token handler using itsdangerous signed tokens (dstok_ prefix).
+    """
+
+    name = "signed"
+
+    async def create_token(
+        self,
+        datasette: "Datasette",
+        actor_id: str,
+        *,
+        expires_after: Optional[int] = None,
+        restrictions: Optional[TokenRestrictions] = None,
+    ) -> str:
+        if not datasette.setting("allow_signed_tokens"):
+            raise ValueError(
+                "Signed tokens are not enabled for this Datasette instance"
+            )
+
+        token = {"a": actor_id, "t": int(time.time())}
+
+        if expires_after:
+            token["d"] = expires_after
+        if restrictions is not None:
+            abbreviated = restrictions.abbreviated(datasette)
+            if abbreviated is not None:
+                token["_r"] = abbreviated
+        return "dstok_{}".format(datasette.sign(token, namespace="token"))
+
+    async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]:
+        prefix = "dstok_"
+
+        if not datasette.setting("allow_signed_tokens"):
+            return None
+
+        max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl")
+
+        if not token.startswith(prefix):
+            return None
+
+        raw = token[len(prefix) :]
+        try:
+            decoded = datasette.unsign(raw, namespace="token")
+        except itsdangerous.BadSignature:
+            return None
+
+        if "t" not in decoded:
+            return None
+        created = decoded["t"]
+        if not isinstance(created, int):
+            return None
+
+        duration = decoded.get("d")
+        if duration is not None and not isinstance(duration, int):
+            return None
+
+        if (duration is None and max_signed_tokens_ttl) or (
+            duration is not None
+            and max_signed_tokens_ttl
+            and duration > max_signed_tokens_ttl
+        ):
+            duration = max_signed_tokens_ttl
+
+        if duration:
+            if time.time() - created > duration:
+                return None
+
+        actor = {"id": decoded["a"], "token": "dstok"}
+
+        if "_r" in decoded:
+            actor["_r"] = decoded["_r"]
+
+        if duration:
+            actor["token_expires"] = created + duration
+
+        return actor

datasette/utils/__init__.py

@@ -155,9 +155,15 @@ Column = namedtuple(
 functions_marked_as_documented = []
 
 
-def documented(fn):
-    functions_marked_as_documented.append(fn)
-    return fn
+def documented(fn=None, *, label=None):
+    def decorate(fn):
+        fn._datasette_docs_label = label or "internals_utils_{}".format(fn.__name__)
+        functions_marked_as_documented.append(fn)
+        return fn
+
+    if fn is None:
+        return decorate
+    return decorate(fn)
 
 
 @documented
@@ -612,7 +618,10 @@ def get_outbound_foreign_keys(conn, table):
 
 def get_all_foreign_keys(conn):
     tables = [
-        r[0] for r in conn.execute('select name from sqlite_master where type="table"')
+        r[0]
+        for r in conn.execute(
+            'select name from sqlite_master where type="table" order by name'
+        )
     ]
     table_to_foreign_keys = {}
     for table in tables:
@@ -634,6 +643,15 @@ def get_all_foreign_keys(conn):
                 {"other_table": table_name, "column": from_, "other_column": to_}
             )
 
+    # Sort foreign keys for deterministic ordering
+    for table in table_to_foreign_keys:
+        table_to_foreign_keys[table]["incoming"].sort(
+            key=lambda fk: (fk["other_table"], fk["column"], fk["other_column"])
+        )
+        table_to_foreign_keys[table]["outgoing"].sort(
+            key=lambda fk: (fk["other_table"], fk["column"], fk["other_column"])
+        )
+
     return table_to_foreign_keys
 
 
@@ -665,19 +683,22 @@ def detect_fts_sql(table):
                     and sql like '%VIRTUAL TABLE%USING FTS%'
                 )
             )
-    """.format(
-        table=table.replace("'", "''")
-    )
+    """.format(table=table.replace("'", "''"))
 
 
 def detect_json1(conn=None):
+    close_conn = False
     if conn is None:
         conn = sqlite3.connect(":memory:")
+        close_conn = True
     try:
         conn.execute("SELECT json('{}')")
         return True
     except Exception:
         return False
+    finally:
+        if close_conn:
+            conn.close()
 
 
 def table_columns(conn, table):
@@ -694,8 +715,11 @@ def table_column_details(conn, table):
             ).fetchall()
         ]
     else:
-        # Treat hidden as 0 for all columns
+        # First trigger a query against sqlite_master to fix an intermittent
+        # test failure, see https://github.com/simonw/datasette/issues/2632
+        conn.execute("select 1 from sqlite_master limit 1").fetchall()
         return [
+            # Treat hidden as 0 for all columns.
             Column(*(list(r) + [0]))
             for r in conn.execute(
                 f"PRAGMA table_info({escape_sqlite(table)});"
@@ -889,18 +913,26 @@ _infinities = {float("inf"), float("-inf")}
 
 
 def remove_infinites(row):
-    to_check = row
+    """
+    Replace float('inf') and float('-inf') with None in a row.
+
+    Returns the original row object unchanged if no infinities are found.
+    """
     if isinstance(row, dict):
-        to_check = row.values()
-    if not any((c in _infinities) if isinstance(c, float) else 0 for c in to_check):
-        return row
-    if isinstance(row, dict):
-        return {
-            k: (None if (isinstance(v, float) and v in _infinities) else v)
-            for k, v in row.items()
-        }
+        for v in row.values():
+            if isinstance(v, float) and v in _infinities:
+                return {
+                    k: (None if isinstance(v2, float) and v2 in _infinities else v2)
+                    for k, v2 in row.items()
+                }
     else:
-        return [None if (isinstance(c, float) and c in _infinities) else c for c in row]
+        for v in row:
+            if isinstance(v, float) and v in _infinities:
+                return [
+                    None if isinstance(v2, float) and v2 in _infinities else v2
+                    for v2 in row
+                ]
+    return row
 
 
 class StaticMount(click.ParamType):
@@ -1065,12 +1097,35 @@ def _gather_arguments(fn, kwargs):
     return call_with
 
 
+@documented
 def call_with_supported_arguments(fn, **kwargs):
+    """
+    Call ``fn`` with the subset of ``**kwargs`` matching its signature.
+
+    This implements dependency injection: the caller provides all available
+    keyword arguments and the function receives only the ones it declares
+    as parameters.
+
+    :param fn: A callable (sync function)
+    :param kwargs: All available keyword arguments
+    :returns: The return value of ``fn``
+    """
     call_with = _gather_arguments(fn, kwargs)
     return fn(*call_with)
 
 
+@documented
 async def async_call_with_supported_arguments(fn, **kwargs):
+    """
+    Async version of :func:`call_with_supported_arguments`.
+
+    Calls ``await fn(...)`` with the subset of ``**kwargs`` matching its
+    signature.
+
+    :param fn: An async callable
+    :param kwargs: All available keyword arguments
+    :returns: The return value of ``await fn(...)``
+    """
     call_with = _gather_arguments(fn, kwargs)
     return await fn(*call_with)
 

datasette/utils/actions_sql.py

@@ -147,7 +147,9 @@ async def _build_single_action_sql(
         raise ValueError(f"Unknown action: {action}")
 
     # Get base resources SQL from the resource class
-    base_resources_sql = await action_obj.resource_class.resources_sql(datasette)
+    base_resources_sql = await action_obj.resource_class.resources_sql(
+        datasette, actor=actor
+    )
 
     permission_sqls = await gather_permission_sql_from_hooks(
         datasette=datasette,
@@ -155,18 +157,36 @@ async def _build_single_action_sql(
         action=action,
     )
 
+    # If permission_sqls is the sentinel, skip all permission checks
+    # Return SQL that allows all resources
+    from datasette.utils.permissions import SKIP_PERMISSION_CHECKS
+
+    if permission_sqls is SKIP_PERMISSION_CHECKS:
+        cols = "parent, child, 'skip_permission_checks' AS reason"
+        if include_is_private:
+            cols += ", 0 AS is_private"
+        return f"SELECT {cols} FROM ({base_resources_sql})", {}
+
     all_params = {}
     rule_sqls = []
+    restriction_sqls = []
 
     for permission_sql in permission_sqls:
-        rule_sqls.append(
-            f"""
+        # Always collect params (even from restriction-only plugins)
+        all_params.update(permission_sql.params or {})
+
+        # Collect restriction SQL filters
+        if permission_sql.restriction_sql:
+            restriction_sqls.append(permission_sql.restriction_sql)
+
+        # Skip plugins that only provide restriction_sql (no permission rules)
+        if permission_sql.sql is None:
+            continue
+        rule_sqls.append(f"""
             SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
-            """.strip()
-        )
-        all_params.update(permission_sql.params or {})
+            """.strip())
 
     # If no rules, return empty result (deny all)
     if not rule_sqls:
@@ -200,6 +220,9 @@ async def _build_single_action_sql(
         anon_params = {}
 
         for permission_sql in anon_permission_sqls:
+            # Skip plugins that only provide restriction_sql (no permission rules)
+            if permission_sql.sql is None:
+                continue
             rewritten_sql = permission_sql.sql
             for key, value in (permission_sql.params or {}).items():
                 anon_key = f"anon_{key}"
@@ -218,6 +241,14 @@ async def _build_single_action_sql(
                     "),",
                 ]
             )
+        else:
+            query_parts.extend(
+                [
+                    "anon_rules AS (",
+                    "  SELECT NULL AS parent, NULL AS child, 0 AS allow, NULL AS reason WHERE 0",
+                    "),",
+                ]
+            )
 
     # Continue with the cascading logic
     query_parts.extend(
@@ -360,6 +391,17 @@ async def _build_single_action_sql(
 
     query_parts.append(")")
 
+    # Add restriction list CTE if there are restrictions
+    if restriction_sqls:
+        # Wrap each restriction_sql in a subquery to avoid operator precedence issues
+        # with UNION ALL inside the restriction SQL statements
+        restriction_intersect = "\nINTERSECT\n".join(
+            f"SELECT * FROM ({sql})" for sql in restriction_sqls
+        )
+        query_parts.extend(
+            [",", "restriction_list AS (", f"  {restriction_intersect}", ")"]
+        )
+
     # Final SELECT
     select_cols = "parent, child, reason"
     if include_is_private:
@@ -369,6 +411,15 @@ async def _build_single_action_sql(
     query_parts.append("FROM decisions")
     query_parts.append("WHERE is_allowed = 1")
 
+    # Add restriction filter if there are restrictions
+    if restriction_sqls:
+        query_parts.append("""
+  AND EXISTS (
+    SELECT 1 FROM restriction_list r
+    WHERE (r.parent = decisions.parent OR r.parent IS NULL)
+      AND (r.child = decisions.child OR r.child IS NULL)
+  )""")
+
     # Add parent filter if specified
     if parent is not None:
         query_parts.append("  AND parent = :filter_parent")
@@ -401,26 +452,47 @@ async def build_permission_rules_sql(
         action=action,
     )
 
+    # If permission_sqls is the sentinel, skip all permission checks
+    # Return SQL that allows everything
+    from datasette.utils.permissions import SKIP_PERMISSION_CHECKS
+
+    if permission_sqls is SKIP_PERMISSION_CHECKS:
+        return (
+            "SELECT NULL AS parent, NULL AS child, 1 AS allow, 'skip_permission_checks' AS reason, 'skip' AS source_plugin",
+            {},
+            [],
+        )
+
     if not permission_sqls:
         return (
             "SELECT NULL AS parent, NULL AS child, 0 AS allow, NULL AS reason, NULL AS source_plugin WHERE 0",
             {},
+            [],
         )
 
     union_parts = []
     all_params = {}
+    restriction_sqls = []
+
     for permission_sql in permission_sqls:
-        union_parts.append(
-            f"""
+        all_params.update(permission_sql.params or {})
+
+        # Collect restriction SQL filters
+        if permission_sql.restriction_sql:
+            restriction_sqls.append(permission_sql.restriction_sql)
+
+        # Skip plugins that only provide restriction_sql (no permission rules)
+        if permission_sql.sql is None:
+            continue
+
+        union_parts.append(f"""
             SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
-            """.strip()
-        )
-        all_params.update(permission_sql.params or {})
+            """.strip())
 
     rules_union = " UNION ALL ".join(union_parts)
-    return rules_union, all_params
+    return rules_union, all_params, restriction_sqls
 
 
 async def check_permission_for_resource(
@@ -447,7 +519,9 @@ async def check_permission_for_resource(
     This builds the cascading permission query and checks if the specific
     resource is in the allowed set.
     """
-    rules_union, all_params = await build_permission_rules_sql(datasette, actor, action)
+    rules_union, all_params, restriction_sqls = await build_permission_rules_sql(
+        datasette, actor, action
+    )
 
     # If no rules (empty SQL), default deny
     if not rules_union:
@@ -457,43 +531,57 @@ async def check_permission_for_resource(
     all_params["_check_parent"] = parent
     all_params["_check_child"] = child
 
+    # If there are restriction filters, check if the resource passes them first
+    if restriction_sqls:
+        # Check if resource is in restriction allowlist
+        # Database-level restrictions (parent, NULL) should match all children (parent, *)
+        # Wrap each restriction_sql in a subquery to avoid operator precedence issues
+        restriction_check = "\nINTERSECT\n".join(
+            f"SELECT * FROM ({sql})" for sql in restriction_sqls
+        )
+        restriction_query = f"""
+WITH restriction_list AS (
+    {restriction_check}
+)
+SELECT EXISTS (
+    SELECT 1 FROM restriction_list
+    WHERE (parent = :_check_parent OR parent IS NULL)
+      AND (child = :_check_child OR child IS NULL)
+) AS in_allowlist
+"""
+        result = await datasette.get_internal_database().execute(
+            restriction_query, all_params
+        )
+        if result.rows and not result.rows[0][0]:
+            # Resource not in restriction allowlist - deny
+            return False
+
     query = f"""
 WITH
 all_rules AS (
   {rules_union}
 ),
-child_lvl AS (
-  SELECT
-         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,
-         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow
+matched_rules AS (
+  SELECT ar.*,
+    CASE
+      WHEN ar.child IS NOT NULL THEN 2  -- child-level (most specific)
+      WHEN ar.parent IS NOT NULL THEN 1  -- parent-level
+      ELSE 0                             -- root/global
+    END AS depth
   FROM all_rules ar
-  WHERE ar.parent = :_check_parent AND ar.child = :_check_child
+  WHERE (ar.parent IS NULL OR ar.parent = :_check_parent)
+    AND (ar.child IS NULL OR ar.child = :_check_child)
 ),
-parent_lvl AS (
-  SELECT
-         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,
-         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow
-  FROM all_rules ar
-  WHERE ar.parent = :_check_parent AND ar.child IS NULL
-),
-global_lvl AS (
-  SELECT
-         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,
-         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow
-  FROM all_rules ar
-  WHERE ar.parent IS NULL AND ar.child IS NULL
+winner AS (
+  SELECT *
+  FROM matched_rules
+  ORDER BY
+    depth DESC,                          -- specificity first (higher depth wins)
+    CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow
+    source_plugin                        -- stable tie-break
+  LIMIT 1
 )
-SELECT
-  CASE
-    WHEN cl.any_deny = 1 THEN 0
-    WHEN cl.any_allow = 1 THEN 1
-    WHEN pl.any_deny = 1 THEN 0
-    WHEN pl.any_allow = 1 THEN 1
-    WHEN gl.any_deny = 1 THEN 0
-    WHEN gl.any_allow = 1 THEN 1
-    ELSE 0
-  END AS is_allowed
-FROM child_lvl cl, parent_lvl pl, global_lvl gl
+SELECT COALESCE((SELECT allow FROM winner), 0) AS is_allowed
 """
 
     # Execute the query against the internal database

datasette/utils/asgi.py

@@ -1,11 +1,28 @@
 import json
+from typing import Optional
 from datasette.utils import MultiParams, calculate_etag
+from datasette.utils.multipart import (
+    parse_form_data,
+    MultipartParseError,
+    FormData,
+    DEFAULT_MAX_FILE_SIZE,
+    DEFAULT_MAX_REQUEST_SIZE,
+    DEFAULT_MAX_FIELDS,
+    DEFAULT_MAX_FILES,
+    DEFAULT_MAX_PARTS,
+    DEFAULT_MAX_FIELD_SIZE,
+    DEFAULT_MAX_MEMORY_FILE_SIZE,
+    DEFAULT_MAX_PART_HEADER_BYTES,
+    DEFAULT_MAX_PART_HEADER_LINES,
+    DEFAULT_MIN_FREE_DISK_BYTES,
+)
 from mimetypes import guess_type
 from urllib.parse import parse_qs, urlunparse, parse_qsl
 from pathlib import Path
 from http.cookies import SimpleCookie, Morsel
 import aiofiles
 import aiofiles.os
+import re
 
 # Workaround for adding samesite support to pre 3.8 python
 Morsel._reserved["samesite"] = "SameSite"
@@ -138,6 +155,71 @@ class Request:
         body = await self.post_body()
         return dict(parse_qsl(body.decode("utf-8"), keep_blank_values=True))
 
+    async def form(
+        self,
+        files: bool = False,
+        max_file_size: int = DEFAULT_MAX_FILE_SIZE,
+        max_request_size: int = DEFAULT_MAX_REQUEST_SIZE,
+        max_fields: int = DEFAULT_MAX_FIELDS,
+        max_files: int = DEFAULT_MAX_FILES,
+        max_parts: Optional[int] = DEFAULT_MAX_PARTS,
+        max_field_size: int = DEFAULT_MAX_FIELD_SIZE,
+        max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE,
+        max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES,
+        max_part_header_lines: int = DEFAULT_MAX_PART_HEADER_LINES,
+        min_free_disk_bytes: int = DEFAULT_MIN_FREE_DISK_BYTES,
+    ) -> FormData:
+        """
+        Parse form data from the request body.
+
+        Supports both application/x-www-form-urlencoded and multipart/form-data.
+
+        Args:
+            files: If True, store file uploads; if False (default), discard them
+            max_file_size: Maximum size per file in bytes (default 50MB)
+            max_request_size: Maximum total request size in bytes (default 100MB)
+            max_fields: Maximum number of form fields (default 1000)
+            max_files: Maximum number of file uploads (default 100)
+            max_parts: Maximum number of multipart parts (default max_fields + max_files)
+            max_field_size: Maximum size of a text field value in bytes (default 100KB)
+            max_memory_file_size: Threshold before files spill to disk (default 1MB)
+            max_part_header_bytes: Maximum bytes allowed in part headers (default 16KB)
+            max_part_header_lines: Maximum header lines per part (default 100)
+            min_free_disk_bytes: Minimum free bytes required in temp dir (default 50MB)
+
+        Returns:
+            FormData object with dict-like access to fields and files.
+            Use form["key"] for first value, form.getlist("key") for all values.
+
+        Raises:
+            BadRequest: If content-type is missing, unsupported, or parsing fails
+        """
+        content_type = self.headers.get("content-type", "")
+        if not content_type:
+            raise BadRequest(
+                "Missing Content-Type header; expected application/x-www-form-urlencoded "
+                "or multipart/form-data"
+            )
+
+        try:
+            return await parse_form_data(
+                receive=self.receive,
+                content_type=content_type,
+                files=files,
+                max_file_size=max_file_size,
+                max_request_size=max_request_size,
+                max_fields=max_fields,
+                max_files=max_files,
+                max_parts=max_parts,
+                max_field_size=max_field_size,
+                max_memory_file_size=max_memory_file_size,
+                max_part_header_bytes=max_part_header_bytes,
+                max_part_header_lines=max_part_header_lines,
+                min_free_disk_bytes=min_free_disk_bytes,
+            )
+        except MultipartParseError as e:
+            raise BadRequest(str(e))
+
     @classmethod
     def fake(cls, path_with_query_string, method="GET", scheme="http", url_vars=None):
         """Useful for constructing Request objects for tests"""
@@ -248,6 +330,9 @@ async def asgi_send_html(send, html, status=200, headers=None):
 
 
 async def asgi_send_redirect(send, location, status=302):
+    # Prevent open redirect vulnerability: strip multiple leading slashes
+    # //example.com would be interpreted as a protocol-relative URL (e.g., https://example.com/)
+    location = re.sub(r"^/+", "/", location)
     await asgi_send(
         send,
         "",

datasette/utils/internal_db.py

@@ -3,8 +3,7 @@ from datasette.utils import table_column_details
 
 
 async def init_internal_db(db):
-    create_tables_sql = textwrap.dedent(
-        """
+    create_tables_sql = textwrap.dedent("""
     CREATE TABLE IF NOT EXISTS catalog_databases (
         database_name TEXT PRIMARY KEY,
         path TEXT,
@@ -68,16 +67,13 @@ async def init_internal_db(db):
         FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name),
         FOREIGN KEY (database_name, table_name) REFERENCES catalog_tables(database_name, table_name)
     );
-    """
-    ).strip()
+    """).strip()
     await db.execute_write_script(create_tables_sql)
     await initialize_metadata_tables(db)
 
 
 async def initialize_metadata_tables(db):
-    await db.execute_write_script(
-        textwrap.dedent(
-            """
+    await db.execute_write_script(textwrap.dedent("""
         CREATE TABLE IF NOT EXISTS metadata_instance (
             key text,
             value text,
@@ -107,9 +103,38 @@ async def initialize_metadata_tables(db):
             value text,
             unique(database_name, resource_name, column_name, key)
         );
-            """
-        )
-    )
+
+        CREATE TABLE IF NOT EXISTS column_types (
+            database_name TEXT NOT NULL,
+            resource_name TEXT NOT NULL,
+            column_name TEXT NOT NULL,
+            column_type TEXT NOT NULL,
+            config TEXT,
+            PRIMARY KEY (database_name, resource_name, column_name)
+        );
+
+        CREATE TABLE IF NOT EXISTS queries (
+            database_name TEXT NOT NULL,
+            name TEXT NOT NULL,
+            sql TEXT NOT NULL,
+            title TEXT,
+            description TEXT,
+            description_html TEXT,
+            options TEXT NOT NULL DEFAULT '{}',
+            parameters TEXT NOT NULL DEFAULT '[]',
+            is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)),
+            is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)),
+            is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)),
+            source TEXT NOT NULL DEFAULT 'user',
+            owner_id TEXT,
+            created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+            updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+            PRIMARY KEY (database_name, name)
+        );
+
+        CREATE INDEX IF NOT EXISTS queries_owner_idx
+            ON queries(owner_id);
+            """))
 
 
 async def populate_schema_tables(internal_db, db):

datasette/utils/multipart.py

@@ -0,0 +1,757 @@
+"""
+Streaming multipart/form-data parser for ASGI applications.
+
+Supports:
+- Streaming parsing without buffering entire body in memory
+- Files spill to disk above configurable threshold
+- Security limits on request size, file size, field count
+- Both multipart/form-data and application/x-www-form-urlencoded
+"""
+
+import asyncio
+import shutil
+import tempfile
+from dataclasses import dataclass, field
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Union,
+)
+from urllib.parse import parse_qsl
+
+# Centralized defaults for multipart/form-data parsing
+DEFAULT_MAX_FILE_SIZE = 50 * 1024 * 1024  # 50MB
+DEFAULT_MAX_REQUEST_SIZE = 100 * 1024 * 1024  # 100MB
+DEFAULT_MAX_FIELDS = 1000
+DEFAULT_MAX_FILES = 100
+# If max_parts is not specified, it defaults to max_fields + max_files
+DEFAULT_MAX_PARTS: Optional[int] = None
+DEFAULT_MAX_FIELD_SIZE = 100 * 1024  # 100KB
+DEFAULT_MAX_MEMORY_FILE_SIZE = 1024 * 1024  # 1MB
+DEFAULT_MAX_PART_HEADER_BYTES = 16 * 1024  # 16KB
+DEFAULT_MAX_PART_HEADER_LINES = 100
+DEFAULT_MIN_FREE_DISK_BYTES = 50 * 1024 * 1024  # 50MB
+
+
+class MultipartParseError(Exception):
+    """Raised when multipart parsing fails."""
+
+    pass
+
+
+@dataclass
+class UploadedFile:
+    """
+    Represents an uploaded file from a multipart form.
+
+    Attributes:
+        name: The form field name
+        filename: The original filename from the upload
+        content_type: The MIME type of the file
+        size: Size in bytes
+    """
+
+    name: str
+    filename: str
+    content_type: Optional[str]
+    size: int
+    _file: tempfile.SpooledTemporaryFile = field(repr=False)
+
+    async def read(self, size: int = -1) -> bytes:
+        """Read file contents."""
+        return await asyncio.to_thread(self._file.read, size)
+
+    async def seek(self, offset: int, whence: int = 0) -> int:
+        """Seek to position in file."""
+        return await asyncio.to_thread(self._file.seek, offset, whence)
+
+    async def close(self) -> None:
+        """Close the underlying file."""
+        await asyncio.to_thread(self._file.close)
+
+    def close_sync(self) -> None:
+        """Close the underlying file synchronously."""
+        self._file.close()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        await self.close()
+
+    def __del__(self):
+        try:
+            self._file.close()
+        except Exception:
+            pass
+
+
+class FormData:
+    """
+    Container for parsed form data, supporting both fields and files.
+
+    Provides dict-like access with support for multiple values per key.
+    """
+
+    def __init__(self):
+        self._data: List[Tuple[str, Union[str, UploadedFile]]] = []
+
+    def append(self, key: str, value: Union[str, UploadedFile]) -> None:
+        """Add a key-value pair."""
+        self._data.append((key, value))
+
+    def __getitem__(self, key: str) -> Union[str, UploadedFile]:
+        """Get the first value for a key."""
+        for k, v in self._data:
+            if k == key:
+                return v
+        raise KeyError(key)
+
+    def get(self, key: str, default: Any = None) -> Optional[Union[str, UploadedFile]]:
+        """Get the first value for a key, or default if not found."""
+        try:
+            return self[key]
+        except KeyError:
+            return default
+
+    def getlist(self, key: str) -> List[Union[str, UploadedFile]]:
+        """Get all values for a key."""
+        return [v for k, v in self._data if k == key]
+
+    def __contains__(self, key: str) -> bool:
+        """Check if key exists."""
+        return any(k == key for k, _ in self._data)
+
+    def __len__(self) -> int:
+        """Return number of items."""
+        return len(self._data)
+
+    def __iter__(self):
+        """Iterate over unique keys."""
+        seen = set()
+        for k, _ in self._data:
+            if k not in seen:
+                seen.add(k)
+                yield k
+
+    def keys(self):
+        """Return unique keys."""
+        return list(self)
+
+    def items(self) -> List[Tuple[str, Union[str, UploadedFile]]]:
+        """Return all key-value pairs."""
+        return list(self._data)
+
+    def values(self) -> List[Union[str, UploadedFile]]:
+        """Return all values."""
+        return [v for _, v in self._data]
+
+    def _uploaded_files(self) -> List[UploadedFile]:
+        """Return UploadedFile instances contained in this form."""
+        return [v for _, v in self._data if isinstance(v, UploadedFile)]
+
+    def close(self) -> None:
+        """
+        Close any uploaded files.
+
+        This provides deterministic cleanup for spooled temp files.
+        """
+        for uploaded in self._uploaded_files():
+            try:
+                uploaded.close_sync()
+            except Exception:
+                # Best-effort cleanup; ignore close errors
+                pass
+
+    async def aclose(self) -> None:
+        """Asynchronously close any uploaded files."""
+        for uploaded in self._uploaded_files():
+            try:
+                await uploaded.close()
+            except Exception:
+                # Best-effort cleanup; ignore close errors
+                pass
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc, tb):
+        self.close()
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc_type, exc, tb):
+        await self.aclose()
+
+
+def parse_content_disposition(header: str) -> Dict[str, Optional[str]]:
+    """
+    Parse Content-Disposition header value.
+
+    Returns dict with 'name', 'filename' keys (filename may be None).
+    """
+    result: Dict[str, Optional[str]] = {"name": None, "filename": None}
+
+    # Split on semicolons, handling quoted strings
+    parts = []
+    current = ""
+    in_quotes = False
+    i = 0
+    while i < len(header):
+        char = header[i]
+        if char == '"' and (i == 0 or header[i - 1] != "\\"):
+            in_quotes = not in_quotes
+            current += char
+        elif char == ";" and not in_quotes:
+            parts.append(current.strip())
+            current = ""
+        else:
+            current += char
+        i += 1
+    if current.strip():
+        parts.append(current.strip())
+
+    for part in parts[1:]:  # Skip the "form-data" part
+        if "=" not in part:
+            continue
+
+        key, _, value = part.partition("=")
+        key = key.strip().lower()
+        value = value.strip()
+
+        # Handle filename* (RFC 5987 encoding)
+        if key == "filename*":
+            # Format: utf-8''encoded_filename or charset'language'encoded_filename
+            if "'" in value:
+                parts_star = value.split("'", 2)
+                if len(parts_star) >= 3:
+                    # charset = parts_star[0]
+                    # language = parts_star[1]
+                    encoded = parts_star[2]
+                    # URL decode
+                    try:
+                        from urllib.parse import unquote
+
+                        result["filename"] = unquote(encoded, encoding="utf-8")
+                    except Exception:
+                        pass
+            continue
+
+        # Remove quotes if present
+        if value.startswith('"') and value.endswith('"'):
+            value = value[1:-1]
+            # Unescape backslash sequences
+            value = value.replace('\\"', '"').replace("\\\\", "\\")
+
+        if key == "name":
+            result["name"] = value
+        elif key == "filename":
+            # Only set if filename* hasn't already set it
+            if result["filename"] is None:
+                # Strip path components (security)
+                # Handle both Unix and Windows paths
+                value = value.replace("\\", "/")
+                if "/" in value:
+                    value = value.rsplit("/", 1)[-1]
+                result["filename"] = value
+
+    return result
+
+
+def parse_content_type(header: str) -> Tuple[str, Dict[str, str]]:
+    """
+    Parse Content-Type header value.
+
+    Returns (media_type, parameters_dict).
+    """
+    parts = header.split(";")
+    media_type = parts[0].strip().lower()
+    params = {}
+
+    for part in parts[1:]:
+        part = part.strip()
+        if "=" in part:
+            key, _, value = part.partition("=")
+            key = key.strip().lower()
+            value = value.strip()
+            # Remove quotes if present
+            if value.startswith('"') and value.endswith('"'):
+                value = value[1:-1]
+            params[key] = value
+
+    return media_type, params
+
+
+class MultipartParser:
+    """
+    Streaming multipart/form-data parser.
+
+    Processes the body chunk by chunk without loading everything into memory.
+    """
+
+    # Parser states
+    STATE_PREAMBLE = 0
+    STATE_HEADER = 1
+    STATE_BODY = 2
+    STATE_DONE = 3
+
+    def __init__(
+        self,
+        boundary: bytes,
+        max_file_size: int = DEFAULT_MAX_FILE_SIZE,
+        max_request_size: int = DEFAULT_MAX_REQUEST_SIZE,
+        max_fields: int = DEFAULT_MAX_FIELDS,
+        max_files: int = DEFAULT_MAX_FILES,
+        max_parts: Optional[int] = DEFAULT_MAX_PARTS,
+        max_field_size: int = DEFAULT_MAX_FIELD_SIZE,
+        max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE,
+        max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES,
+        max_part_header_lines: int = DEFAULT_MAX_PART_HEADER_LINES,
+        min_free_disk_bytes: int = DEFAULT_MIN_FREE_DISK_BYTES,
+        handle_files: bool = False,
+    ):
+        self.boundary = b"--" + boundary
+        self.end_boundary = self.boundary + b"--"
+        self.max_file_size = max_file_size
+        self.max_request_size = max_request_size
+        self.max_fields = max_fields
+        self.max_files = max_files
+        # If not specified, tie max_parts to the other cardinality limits
+        if max_parts is None:
+            max_parts = max_fields + max_files
+        self.max_parts = max_parts
+        self.max_field_size = max_field_size
+        self.max_memory_file_size = max_memory_file_size
+        self.max_part_header_bytes = max_part_header_bytes
+        self.max_part_header_lines = max_part_header_lines
+        self.min_free_disk_bytes = min_free_disk_bytes
+        self.handle_files = handle_files
+
+        self.state = self.STATE_PREAMBLE
+        self.buffer = bytearray()
+        self.total_bytes = 0
+        self.field_count = 0
+        self.file_count = 0
+        self.part_count = 0
+        self.current_part_size = 0
+        self.current_header_bytes = 0
+        self.current_header_lines = 0
+
+        self.form_data = FormData()
+        self._disk_check_interval_bytes = 1024 * 1024  # 1MB between disk checks
+        self._bytes_since_disk_check = 0
+        self._tempdir = tempfile.gettempdir()
+
+        # Current part state
+        self.current_headers: Dict[str, str] = {}
+        self.current_file: Optional[tempfile.SpooledTemporaryFile] = None
+        self.current_body = bytearray()
+        self.current_name: Optional[str] = None
+        self.current_filename: Optional[str] = None
+        self.current_content_type: Optional[str] = None
+
+    def feed(self, chunk: bytes) -> None:
+        """Feed a chunk of data to the parser."""
+        self.total_bytes += len(chunk)
+        if self.total_bytes > self.max_request_size:
+            raise MultipartParseError("Request body too large")
+
+        self.buffer.extend(chunk)
+        self._process()
+
+    def _process(self) -> None:
+        """Process buffered data."""
+        while True:
+            if self.state == self.STATE_PREAMBLE:
+                if not self._process_preamble():
+                    break
+            elif self.state == self.STATE_HEADER:
+                if not self._process_header():
+                    break
+            elif self.state == self.STATE_BODY:
+                if not self._process_body():
+                    break
+            elif self.state == self.STATE_DONE:
+                break
+
+    def _process_preamble(self) -> bool:
+        """Skip preamble and find first boundary."""
+        # Look for boundary (could be at start or after preamble)
+        # Try both \r\n prefixed and bare boundary at start
+        idx = self.buffer.find(self.boundary)
+        if idx == -1:
+            # Keep potential partial boundary at end
+            keep = len(self.boundary) - 1
+            if len(self.buffer) > keep:
+                self.buffer = self.buffer[-keep:]
+            return False
+
+        # Found boundary, skip to after it
+        after_boundary = idx + len(self.boundary)
+
+        # Check for end boundary
+        if self.buffer[idx : idx + len(self.end_boundary)] == self.end_boundary:
+            self.state = self.STATE_DONE
+            return False
+
+        # Skip CRLF or LF after boundary
+        if after_boundary < len(self.buffer):
+            if self.buffer[after_boundary : after_boundary + 2] == b"\r\n":
+                after_boundary += 2
+            elif self.buffer[after_boundary : after_boundary + 1] == b"\n":
+                after_boundary += 1
+
+        self.buffer = self.buffer[after_boundary:]
+        self.state = self.STATE_HEADER
+        self.current_headers = {}
+        self.current_header_bytes = 0
+        self.current_header_lines = 0
+        return True
+
+    def _process_header(self) -> bool:
+        """Parse part headers."""
+        while True:
+            # Look for end of header line
+            crlf_idx = self.buffer.find(b"\r\n")
+            lf_idx = self.buffer.find(b"\n")
+
+            if crlf_idx == -1 and lf_idx == -1:
+                # Guard against unbounded header buffering if no newline is ever sent
+                if len(self.buffer) > self.max_part_header_bytes:
+                    raise MultipartParseError("Part headers too large")
+                return False  # Need more data
+
+            # Use whichever comes first
+            if crlf_idx != -1 and (lf_idx == -1 or crlf_idx < lf_idx):
+                idx = crlf_idx
+                line_end_len = 2
+            else:
+                idx = lf_idx
+                line_end_len = 1
+
+            line = self.buffer[:idx]
+            self.buffer = self.buffer[idx + line_end_len :]
+
+            self.current_header_lines += 1
+            self.current_header_bytes += idx + line_end_len
+            if (
+                self.current_header_lines > self.max_part_header_lines
+                or self.current_header_bytes > self.max_part_header_bytes
+            ):
+                raise MultipartParseError("Part headers too large")
+
+            if not line:
+                # Empty line = end of headers
+                self._start_body()
+                self.state = self.STATE_BODY
+                return True
+
+            # Parse header
+            try:
+                line_str = line.decode("utf-8", errors="replace")
+            except Exception:
+                line_str = line.decode("latin-1")
+
+            if ":" in line_str:
+                name, _, value = line_str.partition(":")
+                self.current_headers[name.strip().lower()] = value.strip()
+
+    def _start_body(self) -> None:
+        """Initialize body parsing for current part."""
+        self.part_count += 1
+        if self.part_count > self.max_parts:
+            raise MultipartParseError("Too many parts")
+
+        # Parse Content-Disposition
+        cd = self.current_headers.get("content-disposition", "")
+        parsed = parse_content_disposition(cd)
+        self.current_name = parsed.get("name")
+        self.current_filename = parsed.get("filename")
+        self.current_content_type = self.current_headers.get("content-type")
+        self.current_part_size = 0
+
+        if self.current_filename is not None:
+            # It's a file
+            self.file_count += 1
+            if self.file_count > self.max_files:
+                raise MultipartParseError("Too many files")
+            if self.handle_files:
+                self.current_file = tempfile.SpooledTemporaryFile(
+                    max_size=self.max_memory_file_size
+                )
+            else:
+                # Will discard file content
+                self.current_file = None
+        else:
+            # It's a text field
+            self.field_count += 1
+            if self.field_count > self.max_fields:
+                raise MultipartParseError("Too many fields")
+            self.current_body = bytearray()
+            self.current_file = None
+
+        # Check disk space before allocating a spooled temp file
+        if self.current_filename is not None and self.handle_files:
+            self._ensure_disk_space()
+
+    def _process_body(self) -> bool:
+        """Process body data for current part."""
+        # Look for boundary in buffer
+        # Need to handle boundary potentially split across chunks
+
+        # The boundary is preceded by \r\n (or \n for lenient parsing)
+        search_boundary = b"\r\n" + self.boundary
+
+        idx = self.buffer.find(search_boundary)
+        if idx == -1:
+            # Try LF-only boundary (lenient)
+            search_boundary_lf = b"\n" + self.boundary
+            idx = self.buffer.find(search_boundary_lf)
+            if idx != -1:
+                search_boundary = search_boundary_lf
+
+        if idx == -1:
+            # No boundary found yet
+            # Keep potential partial boundary at end of buffer
+            safe_len = len(self.buffer) - len(search_boundary) - 1
+            if safe_len > 0:
+                safe_data = self.buffer[:safe_len]
+                self._write_body_data(bytes(safe_data))
+                self.buffer = self.buffer[safe_len:]
+            return False
+
+        # Found boundary - write remaining body data
+        body_data = self.buffer[:idx]
+        self._write_body_data(bytes(body_data))
+
+        # Move past the boundary
+        after_boundary = idx + len(search_boundary)
+
+        # Check for end boundary
+        remaining = self.buffer[after_boundary:]
+        if remaining.startswith(b"--"):
+            # End boundary
+            self._finish_part()
+            self.state = self.STATE_DONE
+            return False
+
+        # Skip CRLF or LF after boundary
+        if remaining.startswith(b"\r\n"):
+            after_boundary += 2
+        elif remaining.startswith(b"\n"):
+            after_boundary += 1
+
+        self.buffer = self.buffer[after_boundary:]
+        self._finish_part()
+        self.state = self.STATE_HEADER
+        self.current_headers = {}
+        self.current_header_bytes = 0
+        self.current_header_lines = 0
+        return True
+
+    def _write_body_data(self, data: bytes) -> None:
+        """Write data to current part body."""
+        if not data:
+            return
+
+        self.current_part_size += len(data)
+
+        if self.current_filename is not None:
+            # File data
+            if self.current_part_size > self.max_file_size:
+                raise MultipartParseError("File too large")
+            if self.handle_files and self.current_file:
+                self._bytes_since_disk_check += len(data)
+                if self._bytes_since_disk_check >= self._disk_check_interval_bytes:
+                    self._ensure_disk_space()
+                    self._bytes_since_disk_check = 0
+                self.current_file.write(data)
+            # else: discard file data
+        else:
+            # Field data
+            if self.current_part_size > self.max_field_size:
+                raise MultipartParseError("Field value too large")
+            self.current_body.extend(data)
+
+    def _finish_part(self) -> None:
+        """Finalize current part and add to form data."""
+        if self.current_name is None:
+            return
+
+        if self.current_filename is not None:
+            # File
+            if self.handle_files and self.current_file:
+                self.current_file.seek(0)
+                uploaded = UploadedFile(
+                    name=self.current_name,
+                    filename=self.current_filename,
+                    content_type=self.current_content_type,
+                    size=self.current_part_size,
+                    _file=self.current_file,
+                )
+                self.form_data.append(self.current_name, uploaded)
+            # else: file was discarded
+        else:
+            # Text field
+            try:
+                value = bytes(self.current_body).decode("utf-8")
+            except UnicodeDecodeError:
+                value = bytes(self.current_body).decode("latin-1")
+            self.form_data.append(self.current_name, value)
+
+        # Reset part state
+        self.current_file = None
+        self.current_body = bytearray()
+        self.current_name = None
+        self.current_filename = None
+        self.current_content_type = None
+
+    def finalize(self) -> FormData:
+        """Finalize parsing and return form data."""
+        # Process any remaining data
+        self._process()
+        if self.state != self.STATE_DONE:
+            raise MultipartParseError(
+                "Truncated multipart body (missing closing boundary)"
+            )
+        return self.form_data
+
+    def _ensure_disk_space(self) -> None:
+        """
+        Ensure there is enough free space on the temp filesystem.
+
+        This is a best-effort guard against filling the disk with uploads.
+        """
+        if not self.handle_files:
+            return
+        if self.min_free_disk_bytes <= 0:
+            return
+        free_bytes = shutil.disk_usage(self._tempdir).free
+        if free_bytes < self.min_free_disk_bytes:
+            raise MultipartParseError("Insufficient disk space for uploads")
+
+
+async def parse_form_data(
+    receive: Callable,
+    content_type: str,
+    files: bool = False,
+    max_file_size: int = DEFAULT_MAX_FILE_SIZE,
+    max_request_size: int = DEFAULT_MAX_REQUEST_SIZE,
+    max_fields: int = DEFAULT_MAX_FIELDS,
+    max_files: int = DEFAULT_MAX_FILES,
+    max_parts: Optional[int] = DEFAULT_MAX_PARTS,
+    max_field_size: int = DEFAULT_MAX_FIELD_SIZE,
+    max_memory_file_size: int = DEFAULT_MAX_MEMORY_FILE_SIZE,
+    max_part_header_bytes: int = DEFAULT_MAX_PART_HEADER_BYTES,
+    max_part_header_lines: int = DEFAULT_MAX_PART_HEADER_LINES,
+    min_free_disk_bytes: int = DEFAULT_MIN_FREE_DISK_BYTES,
+) -> FormData:
+    """
+    Parse form data from an ASGI receive callable.
+
+    Supports both application/x-www-form-urlencoded and multipart/form-data.
+
+    Args:
+        receive: ASGI receive callable
+        content_type: Content-Type header value
+        files: If True, store file uploads; if False, discard them
+        max_file_size: Maximum size per file in bytes
+        max_request_size: Maximum total request size in bytes
+        max_fields: Maximum number of form fields
+        max_files: Maximum number of file uploads
+        max_field_size: Maximum size of a text field value
+        max_memory_file_size: File size threshold before spilling to disk
+
+    Returns:
+        FormData object containing parsed fields and files
+    """
+    media_type, params = parse_content_type(content_type)
+
+    if media_type == "application/x-www-form-urlencoded":
+        # Read entire body for URL-encoded forms (they're typically small)
+        body = bytearray()
+        total = 0
+        while True:
+            message = await receive()
+            message_type = message.get("type")
+            if message_type == "http.disconnect":
+                raise MultipartParseError("Client disconnected during request body")
+            if message_type is not None and message_type != "http.request":
+                continue
+            chunk = message.get("body", b"")
+            total += len(chunk)
+            if total > max_request_size:
+                raise MultipartParseError("Request body too large")
+            body.extend(chunk)
+            if not message.get("more_body", False):
+                break
+
+        form_data = FormData()
+        try:
+            pairs = parse_qsl(bytes(body).decode("utf-8"), keep_blank_values=True)
+        except UnicodeDecodeError:
+            pairs = parse_qsl(bytes(body).decode("latin-1"), keep_blank_values=True)
+
+        for key, value in pairs:
+            form_data.append(key, value)
+
+        return form_data
+
+    elif media_type == "multipart/form-data":
+        boundary = params.get("boundary")
+        if not boundary:
+            raise MultipartParseError("Missing boundary in Content-Type")
+
+        parser = MultipartParser(
+            boundary=boundary.encode("utf-8"),
+            max_file_size=max_file_size,
+            max_request_size=max_request_size,
+            max_fields=max_fields,
+            max_files=max_files,
+            max_parts=max_parts,
+            max_field_size=max_field_size,
+            max_memory_file_size=max_memory_file_size,
+            max_part_header_bytes=max_part_header_bytes,
+            max_part_header_lines=max_part_header_lines,
+            min_free_disk_bytes=min_free_disk_bytes,
+            handle_files=files,
+        )
+
+        # Stream body through parser
+        batch_target = 64 * 1024
+        batch = bytearray()
+
+        async def flush_batch() -> None:
+            if batch:
+                data = bytes(batch)
+                batch.clear()
+                await asyncio.to_thread(parser.feed, data)
+
+        while True:
+            message = await receive()
+            message_type = message.get("type")
+            if message_type == "http.disconnect":
+                raise MultipartParseError("Client disconnected during request body")
+            if message_type is not None and message_type != "http.request":
+                continue
+            chunk = message.get("body", b"")
+            if chunk:
+                batch.extend(chunk)
+                if len(batch) >= batch_target:
+                    await flush_batch()
+            if not message.get("more_body", False):
+                break
+
+        await flush_batch()
+        return await asyncio.to_thread(parser.finalize)
+
+    else:
+        raise MultipartParseError(
+            f"Unsupported Content-Type: {media_type}. "
+            "Expected application/x-www-form-urlencoded or multipart/form-data"
+        )

datasette/utils/permissions.py

@@ -9,14 +9,26 @@ from datasette.permissions import PermissionSQL
 from datasette.plugins import pm
 from datasette.utils import await_me_maybe
 
+# Sentinel object to indicate permission checks should be skipped
+SKIP_PERMISSION_CHECKS = object()
+
 
 async def gather_permission_sql_from_hooks(
     *, datasette, actor: dict | None, action: str
-) -> List[PermissionSQL]:
+) -> List[PermissionSQL] | object:
     """Collect PermissionSQL objects from the permission_resources_sql hook.
 
     Ensures that each returned PermissionSQL has a populated ``source``.
+
+    Returns SKIP_PERMISSION_CHECKS sentinel if skip_permission_checks context variable
+    is set, signaling that all permission checks should be bypassed.
     """
+    from datasette.permissions import _skip_permission_checks
+
+    # Check if we should skip permission checks BEFORE calling hooks
+    # This avoids creating unawaited coroutines
+    if _skip_permission_checks.get():
+        return SKIP_PERMISSION_CHECKS
 
     hook_caller = pm.hook.permission_resources_sql
     hookimpls = hook_caller.get_hookimpls()
@@ -99,13 +111,15 @@ def build_rules_union(
         # No namespacing - just use plugin params as-is
         params.update(p.params or {})
 
-        parts.append(
-            f"""
+        # Skip plugins that only provide restriction_sql (no permission rules)
+        if p.sql is None:
+            continue
+
+        parts.append(f"""
             SELECT parent, child, allow, reason, '{p.source}' AS source_plugin FROM (
                 {p.sql}
             )
-            """.strip()
-        )
+            """.strip())
 
     if not parts:
         # Empty UNION that returns no rows
@@ -155,6 +169,8 @@ async def resolve_permissions_from_catalog(
       - resource (rendered "/parent/child" or "/parent" or "/")
     """
     resolved_plugins: List[PermissionSQL] = []
+    restriction_sqls: List[str] = []
+
     for plugin in plugins:
         if callable(plugin) and not isinstance(plugin, PermissionSQL):
             resolved = plugin(action)  # type: ignore[arg-type]
@@ -164,6 +180,10 @@ async def resolve_permissions_from_catalog(
             raise TypeError("Plugin providers must return PermissionSQL instances")
         resolved_plugins.append(resolved)
 
+        # Collect restriction SQL filters
+        if resolved.restriction_sql:
+            restriction_sqls.append(resolved.restriction_sql)
+
     union_sql, rule_params = build_rules_union(actor, resolved_plugins)
     all_params = {
         **(candidate_params or {}),
@@ -199,8 +219,8 @@ async def resolve_permissions_from_catalog(
                  PARTITION BY parent, child
                  ORDER BY
                    depth DESC,                          -- specificity first
-                   CASE WHEN allow=0 THEN 0 ELSE 1 END, -- deny over allow at same depth
-                   source_plugin                         -- stable tie-break
+                   CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow at same depth
+                   source_plugin                        -- stable tie-break
                ) AS rn
         FROM matched
     ),
@@ -228,6 +248,145 @@ async def resolve_permissions_from_catalog(
     ORDER BY c.parent, c.child
     """
 
+    # If there are restriction filters, wrap the query with INTERSECT
+    # This ensures only resources in the restriction allowlist are returned
+    if restriction_sqls:
+        # Start with the main query, but select only parent/child for the INTERSECT
+        main_query_for_intersect = f"""
+        WITH
+        cands AS (
+            {candidate_sql}
+        ),
+        rules AS (
+            {union_sql}
+        ),
+        matched AS (
+            SELECT
+                c.parent, c.child,
+                r.allow, r.reason, r.source_plugin,
+                CASE
+                  WHEN r.child  IS NOT NULL THEN 2  -- child-level (most specific)
+                  WHEN r.parent IS NOT NULL THEN 1  -- parent-level
+                  ELSE 0                            -- root/global
+                END AS depth
+            FROM cands c
+            JOIN rules r
+              ON (r.parent IS NULL OR r.parent = c.parent)
+             AND (r.child  IS NULL OR r.child  = c.child)
+        ),
+        ranked AS (
+            SELECT *,
+                   ROW_NUMBER() OVER (
+                     PARTITION BY parent, child
+                     ORDER BY
+                       depth DESC,                          -- specificity first
+                       CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow at same depth
+                       source_plugin                        -- stable tie-break
+                   ) AS rn
+            FROM matched
+        ),
+        winner AS (
+            SELECT parent, child,
+                   allow, reason, source_plugin, depth
+            FROM ranked WHERE rn = 1
+        ),
+        permitted_resources AS (
+            SELECT c.parent, c.child
+            FROM cands c
+            LEFT JOIN winner w
+              ON ((w.parent = c.parent) OR (w.parent IS NULL AND c.parent IS NULL))
+             AND ((w.child  = c.child ) OR (w.child  IS NULL AND c.child  IS NULL))
+            WHERE COALESCE(w.allow, CASE WHEN :implicit_deny THEN 0 ELSE NULL END) = 1
+        )
+        SELECT parent, child FROM permitted_resources
+        """
+
+        # Build restriction list with INTERSECT (all must match)
+        # Then filter to resources that match hierarchically
+        # Wrap each restriction_sql in a subquery to avoid operator precedence issues
+        # with UNION ALL inside the restriction SQL statements
+        restriction_intersect = "\nINTERSECT\n".join(
+            f"SELECT * FROM ({sql})" for sql in restriction_sqls
+        )
+
+        # Combine: resources allowed by permissions AND in restriction allowlist
+        # Database-level restrictions (parent, NULL) should match all children (parent, *)
+        filtered_resources = f"""
+        WITH restriction_list AS (
+            {restriction_intersect}
+        ),
+        permitted AS (
+            {main_query_for_intersect}
+        ),
+        filtered AS (
+            SELECT p.parent, p.child
+            FROM permitted p
+            WHERE EXISTS (
+                SELECT 1 FROM restriction_list r
+                WHERE (r.parent = p.parent OR r.parent IS NULL)
+                  AND (r.child = p.child OR r.child IS NULL)
+            )
+        )
+        """
+
+        # Now join back to get full results for only the filtered resources
+        sql = f"""
+        {filtered_resources}
+        , cands AS (
+            {candidate_sql}
+        ),
+        rules AS (
+            {union_sql}
+        ),
+        matched AS (
+            SELECT
+                c.parent, c.child,
+                r.allow, r.reason, r.source_plugin,
+                CASE
+                  WHEN r.child  IS NOT NULL THEN 2  -- child-level (most specific)
+                  WHEN r.parent IS NOT NULL THEN 1  -- parent-level
+                  ELSE 0                            -- root/global
+                END AS depth
+            FROM cands c
+            JOIN rules r
+              ON (r.parent IS NULL OR r.parent = c.parent)
+             AND (r.child  IS NULL OR r.child  = c.child)
+        ),
+        ranked AS (
+            SELECT *,
+                   ROW_NUMBER() OVER (
+                     PARTITION BY parent, child
+                     ORDER BY
+                       depth DESC,                          -- specificity first
+                       CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow at same depth
+                       source_plugin                        -- stable tie-break
+                   ) AS rn
+            FROM matched
+        ),
+        winner AS (
+            SELECT parent, child,
+                   allow, reason, source_plugin, depth
+            FROM ranked WHERE rn = 1
+        )
+        SELECT
+          c.parent, c.child,
+          COALESCE(w.allow, CASE WHEN :implicit_deny THEN 0 ELSE NULL END) AS allow,
+          COALESCE(w.reason, CASE WHEN :implicit_deny THEN 'implicit deny' ELSE NULL END) AS reason,
+          w.source_plugin,
+          COALESCE(w.depth, -1) AS depth,
+          :action AS action,
+          CASE
+            WHEN c.parent IS NULL THEN '/'
+            WHEN c.child  IS NULL THEN '/' || c.parent
+            ELSE '/' || c.parent || '/' || c.child
+          END AS resource
+        FROM filtered c
+        LEFT JOIN winner w
+          ON ((w.parent = c.parent) OR (w.parent IS NULL AND c.parent IS NULL))
+         AND ((w.child  = c.child ) OR (w.child  IS NULL AND c.child  IS NULL))
+        ORDER BY c.parent, c.child
+        """
+
     rows_iter: Iterable[sqlite3.Row] = await db.execute(
         sql,
         {**all_params, "implicit_deny": 1 if implicit_deny else 0},

datasette/utils/sql_analysis.py

@@ -0,0 +1,99 @@
+from dataclasses import dataclass
+from typing import Literal
+
+from datasette.utils.sqlite import sqlite3
+
+SQLTableOperation = Literal["read", "insert", "update", "delete"]
+
+
+@dataclass(frozen=True)
+class SQLTableAccess:
+    operation: SQLTableOperation
+    database: str | None
+    table: str
+    sqlite_schema: str | None
+    columns: tuple[str, ...] = ()
+    source: str | None = None
+
+
+@dataclass(frozen=True)
+class SQLAnalysis:
+    table_accesses: tuple[SQLTableAccess, ...]
+
+
+_ACTION_TO_OPERATION: dict[int, SQLTableOperation] = {
+    sqlite3.SQLITE_READ: "read",
+    sqlite3.SQLITE_INSERT: "insert",
+    sqlite3.SQLITE_UPDATE: "update",
+    sqlite3.SQLITE_DELETE: "delete",
+}
+
+
+def analyze_sql_tables(
+    conn,
+    sql: str,
+    params=None,
+    *,
+    database_name: str | None = None,
+    schema_to_database: dict[str, str] | None = None,
+) -> SQLAnalysis:
+    """
+    Return tables accessed by a SQL statement according to SQLite's authorizer.
+
+    This function is synchronous and connection-based. It temporarily installs a
+    SQLite authorizer, prepares ``EXPLAIN <sql>``, and returns the table access
+    callbacks observed while SQLite compiles the statement.
+    """
+    accesses: dict[
+        tuple[SQLTableOperation, str | None, str, str | None, str | None], set[str]
+    ] = {}
+
+    def database_for_schema(sqlite_schema):
+        if schema_to_database and sqlite_schema in schema_to_database:
+            return schema_to_database[sqlite_schema]
+        if sqlite_schema == "main" and database_name is not None:
+            return database_name
+        return sqlite_schema
+
+    def authorizer(action, arg1, arg2, sqlite_schema, source):
+        operation = _ACTION_TO_OPERATION.get(action)
+        if operation is None or arg1 is None:
+            return sqlite3.SQLITE_OK
+
+        key = (
+            operation,
+            database_for_schema(sqlite_schema),
+            arg1,
+            sqlite_schema,
+            source,
+        )
+        columns = accesses.setdefault(key, set())
+        if operation in ("read", "update") and arg2 is not None:
+            columns.add(arg2)
+        return sqlite3.SQLITE_OK
+
+    conn.set_authorizer(authorizer)
+    try:
+        conn.execute("EXPLAIN " + sql, params if params is not None else {}).fetchall()
+    finally:
+        conn.set_authorizer(None)
+
+    return SQLAnalysis(
+        table_accesses=tuple(
+            SQLTableAccess(
+                operation=operation,
+                database=database,
+                table=table,
+                sqlite_schema=sqlite_schema,
+                columns=tuple(sorted(columns)),
+                source=source,
+            )
+            for (
+                operation,
+                database,
+                table,
+                sqlite_schema,
+                source,
+            ), columns in accesses.items()
+        )
+    )

datasette/utils/sqlite.py

@@ -20,15 +20,16 @@ def sqlite_version():
 
 
 def _sqlite_version():
-    return tuple(
-        map(
-            int,
-            sqlite3.connect(":memory:")
-            .execute("select sqlite_version()")
-            .fetchone()[0]
-            .split("."),
+    conn = sqlite3.connect(":memory:")
+    try:
+        return tuple(
+            map(
+                int,
+                conn.execute("select sqlite_version()").fetchone()[0].split("."),
+            )
         )
-    )
+    finally:
+        conn.close()
 
 
 def supports_table_xinfo():

datasette/utils/testing.py

@@ -95,15 +95,8 @@ class TestClient:
         cookies = cookies or {}
         post_data = post_data or {}
         assert not (post_data and body), "Provide one or other of body= or post_data="
-        # Maybe fetch a csrftoken first
-        if csrftoken_from is not None:
-            assert body is None, "body= is not compatible with csrftoken_from="
-            if csrftoken_from is True:
-                csrftoken_from = path
-            token_response = await self._request(csrftoken_from, cookies=cookies)
-            csrftoken = token_response.cookies["ds_csrftoken"]
-            cookies["ds_csrftoken"] = csrftoken
-            post_data["csrftoken"] = csrftoken
+        # csrftoken_from is accepted for backward compatibility but is now a no-op.
+        # Datasette no longer uses CSRF tokens - see CrossOriginProtectionMiddleware.
         if post_data:
             body = urlencode(post_data, doseq=True)
         return await self._request(

datasette/version.py

@@ -1,2 +1,2 @@
-__version__ = "1.0a19"
+__version__ = "1.0a30"
 __version_info__ = tuple(__version__.split("."))

datasette/views/base.py

@@ -1,7 +1,6 @@
 import asyncio
 import csv
 import hashlib
-import json
 import sys
 import textwrap
 import time
@@ -242,8 +241,7 @@ class DataView(BaseView):
                 data, extra_template_data, templates = response_or_template_contexts
         except QueryInterrupted as ex:
             raise DatasetteError(
-                textwrap.dedent(
-                    """
+                textwrap.dedent("""
                 <p>SQL query took too long. The time limit is controlled by the
                 <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>
                 configuration option.</p>
@@ -252,10 +250,7 @@ class DataView(BaseView):
                 let ta = document.querySelector("textarea");
                 ta.style.height = ta.scrollHeight + "px";
                 </script>
-            """.format(
-                        escape(ex.sql)
-                    )
-                ).strip(),
+            """.format(escape(ex.sql))).strip(),
                 title="SQL Interrupted",
                 status=400,
                 message_is_html=True,

datasette/views/database.py

@@ -13,6 +13,7 @@ import textwrap
 from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent
 from datasette.database import QueryInterrupted
 from datasette.resources import DatabaseResource, QueryResource
+from datasette.stored_queries import stored_query_to_dict
 from datasette.utils import (
     add_cors_headers,
     await_me_maybe,
@@ -35,6 +36,7 @@ from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden
 from datasette.plugins import pm
 
 from .base import BaseView, DatasetteError, View, _error, stream_csv
+from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns
 from . import Context
 
 
@@ -61,8 +63,10 @@ class DatabaseView(View):
             if request.url_vars.get("format"):
                 redirect_url += "." + request.url_vars.get("format")
             redirect_url += "?" + request.query_string
-            return Response.redirect(redirect_url)
-            return await QueryView()(request, datasette)
+            response = Response.redirect(redirect_url)
+            if datasette.cors:
+                add_cors_headers(response.headers)
+            return response
 
         if format_ not in ("html", "json"):
             raise NotFound("Invalid format: {}".format(format_))
@@ -90,24 +94,19 @@ class DatabaseView(View):
 
         tables = await get_tables(datasette, request, db, allowed_dict)
 
-        # Get allowed queries using the new permission system
-        allowed_query_page = await datasette.allowed_resources(
-            "view-query",
-            request.actor,
-            parent=database,
-            include_is_private=True,
-            limit=1000,
+        queries_page = await datasette.list_queries(
+            database,
+            actor=request.actor,
+            limit=5,
+            include_private=True,
+        )
+        stored_queries = queries_page.queries
+        queries_more = queries_page.has_more
+        queries_count = (
+            await datasette.count_queries(database, actor=request.actor)
+            if queries_more
+            else len(stored_queries)
         )
-
-        # Build canned_queries list by looking up each allowed query
-        all_queries = await datasette.get_canned_queries(database, request.actor)
-        canned_queries = []
-        for query_resource in allowed_query_page.resources:
-            query_name = query_resource.child
-            if query_name in all_queries:
-                canned_queries.append(
-                    dict(all_queries[query_name], private=query_resource.private)
-                )
 
         async def database_actions():
             links = []
@@ -130,6 +129,7 @@ class DatabaseView(View):
             actor=request.actor,
         )
         json_data = {
+            "ok": True,
             "database": database,
             "private": private,
             "path": datasette.urls.database(database),
@@ -137,7 +137,9 @@ class DatabaseView(View):
             "tables": tables,
             "hidden_count": len([t for t in tables if t["hidden"]]),
             "views": sql_views,
-            "queries": canned_queries,
+            "queries": [stored_query_to_dict(query) for query in stored_queries],
+            "queries_more": queries_more,
+            "queries_count": queries_count,
             "allow_execute_sql": allow_execute_sql,
             "table_columns": (
                 await _table_columns(datasette, database) if allow_execute_sql else {}
@@ -170,7 +172,9 @@ class DatabaseView(View):
                     tables=tables,
                     hidden_count=len([t for t in tables if t["hidden"]]),
                     views=sql_views,
-                    queries=canned_queries,
+                    queries=stored_queries,
+                    queries_more=queries_more,
+                    queries_count=queries_count,
                     allow_execute_sql=allow_execute_sql,
                     table_columns=(
                         await _table_columns(datasette, database)
@@ -218,7 +222,11 @@ class DatabaseContext(Context):
     tables: list = field(metadata={"help": "List of table objects in the database"})
     hidden_count: int = field(metadata={"help": "Count of hidden tables"})
     views: list = field(metadata={"help": "List of view objects in the database"})
-    queries: list = field(metadata={"help": "List of canned query objects"})
+    queries: list = field(metadata={"help": "List of stored query objects"})
+    queries_more: bool = field(
+        metadata={"help": "Boolean indicating if more stored queries are available"}
+    )
+    queries_count: int = field(metadata={"help": "Count of visible stored queries"})
     allow_execute_sql: bool = field(
         metadata={"help": "Boolean indicating if custom SQL can be executed"}
     )
@@ -263,8 +271,8 @@ class QueryContext(Context):
     query: dict = field(
         metadata={"help": "The SQL query object containing the `sql` string"}
     )
-    canned_query: str = field(
-        metadata={"help": "The name of the canned query if this is a canned query"}
+    stored_query: str = field(
+        metadata={"help": "The name of the stored query if this is a stored query"}
     )
     private: bool = field(
         metadata={"help": "Boolean indicating if this is a private database"}
@@ -272,13 +280,13 @@ class QueryContext(Context):
     # urls: dict = field(
     #     metadata={"help": "Object containing URL helpers like `database()`"}
     # )
-    canned_query_write: bool = field(
+    stored_query_write: bool = field(
         metadata={
-            "help": "Boolean indicating if this is a canned query that allows writes"
+            "help": "Boolean indicating if this is a stored query that allows writes"
         }
     )
     metadata: dict = field(
-        metadata={"help": "Metadata about the database or the canned query"}
+        metadata={"help": "Metadata about the database or the stored query"}
     )
     db_is_immutable: bool = field(
         metadata={"help": "Boolean indicating if this database is immutable"}
@@ -299,12 +307,15 @@ class QueryContext(Context):
     allow_execute_sql: bool = field(
         metadata={"help": "Boolean indicating if custom SQL can be executed"}
     )
+    save_query_url: str = field(
+        metadata={"help": "URL to save the current arbitrary SQL as a query"}
+    )
     tables: list = field(metadata={"help": "List of table objects in the database"})
     named_parameter_values: dict = field(
         metadata={"help": "Dictionary of parameter names/values"}
     )
     edit_sql_url: str = field(
-        metadata={"help": "URL to edit the SQL for a canned query"}
+        metadata={"help": "URL to edit the SQL for a stored query"}
     )
     display_rows: list = field(metadata={"help": "List of result rows to display"})
     columns: list = field(metadata={"help": "List of column names"})
@@ -328,8 +339,8 @@ class QueryContext(Context):
     top_query: callable = field(
         metadata={"help": "Callable to render the top_query slot"}
     )
-    top_canned_query: callable = field(
-        metadata={"help": "Callable to render the top_canned_query slot"}
+    top_stored_query: callable = field(
+        metadata={"help": "Callable to render the top_stored_query slot"}
     )
     query_actions: callable = field(
         metadata={
@@ -420,21 +431,32 @@ class QueryView(View):
 
         db = await datasette.resolve_database(request)
 
-        # We must be a canned query
+        # We must be a stored query
         table_found = False
         try:
             await datasette.resolve_table(request)
             table_found = True
         except TableNotFound as table_not_found:
-            canned_query = await datasette.get_canned_query(
-                table_not_found.database_name, table_not_found.table, request.actor
+            stored_query = await datasette.get_query(
+                table_not_found.database_name, table_not_found.table
             )
-            if canned_query is None:
+            if stored_query is None:
                 raise
         if table_found:
             # That should not have happened
             raise DatasetteError("Unexpected table found on POST", status=404)
 
+        if not await datasette.allowed(
+            action="view-query",
+            resource=QueryResource(database=db.name, query=stored_query.name),
+            actor=request.actor,
+        ):
+            raise Forbidden("You do not have permission to view this query")
+
+        await _ensure_stored_query_execution_permissions(
+            datasette, db, stored_query, request.actor
+        )
+
         # If database is immutable, return an error
         if not db.is_mutable:
             raise Forbidden("Database is immutable")
@@ -459,18 +481,18 @@ class QueryView(View):
             or request.args.get("_json")
             or params.get("_json")
         )
-        params_for_query = MagicParameters(
-            canned_query["sql"], params, request, datasette
-        )
+        params_for_query = MagicParameters(stored_query.sql, params, request, datasette)
         await params_for_query.execute_params()
         ok = None
         redirect_url = None
         try:
-            cursor = await db.execute_write(canned_query["sql"], params_for_query)
+            cursor = await db.execute_write(
+                stored_query.sql, params_for_query, request=request
+            )
             # success message can come from on_success_message or on_success_message_sql
             message = None
             message_type = datasette.INFO
-            on_success_message_sql = canned_query.get("on_success_message_sql")
+            on_success_message_sql = stored_query.on_success_message_sql
             if on_success_message_sql:
                 try:
                     message_result = (
@@ -482,18 +504,19 @@ class QueryView(View):
                     message = "Error running on_success_message_sql: {}".format(ex)
                     message_type = datasette.ERROR
             if not message:
-                message = canned_query.get(
-                    "on_success_message"
-                ) or "Query executed, {} row{} affected".format(
-                    cursor.rowcount, "" if cursor.rowcount == 1 else "s"
+                message = (
+                    stored_query.on_success_message
+                    or "Query executed, {} row{} affected".format(
+                        cursor.rowcount, "" if cursor.rowcount == 1 else "s"
+                    )
                 )
 
-            redirect_url = canned_query.get("on_success_redirect")
+            redirect_url = stored_query.on_success_redirect
             ok = True
         except Exception as ex:
-            message = canned_query.get("on_error_message") or str(ex)
+            message = stored_query.on_error_message or str(ex)
             message_type = datasette.ERROR
-            redirect_url = canned_query.get("on_error_redirect")
+            redirect_url = stored_query.on_error_redirect
             ok = False
         if should_return_json:
             return Response.json(
@@ -526,31 +549,35 @@ class QueryView(View):
         # Create lookup dict for quick access
         allowed_dict = {r.child: r for r in allowed_tables_page.resources}
 
-        # Are we a canned query?
-        canned_query = None
-        canned_query_write = False
+        # Are we a stored query?
+        stored_query = None
+        stored_query_write = False
         if "table" in request.url_vars:
             try:
                 await datasette.resolve_table(request)
             except TableNotFound as table_not_found:
-                # Was this actually a canned query?
-                canned_query = await datasette.get_canned_query(
-                    table_not_found.database_name, table_not_found.table, request.actor
+                # Was this actually a stored query?
+                stored_query = await datasette.get_query(
+                    table_not_found.database_name, table_not_found.table
                 )
-                if canned_query is None:
+                if stored_query is None:
                     raise
-                canned_query_write = bool(canned_query.get("write"))
+                stored_query_write = stored_query.is_write
 
         private = False
-        if canned_query:
-            # Respect canned query permissions
+        if stored_query:
+            # Respect stored query permissions
             visible, private = await datasette.check_visibility(
                 request.actor,
                 action="view-query",
-                resource=QueryResource(database=database, query=canned_query["name"]),
+                resource=QueryResource(database=database, query=stored_query.name),
             )
             if not visible:
                 raise Forbidden("You do not have permission to view this query")
+            if not stored_query_write:
+                await _ensure_stored_query_execution_permissions(
+                    datasette, db, stored_query, request.actor
+                )
 
         else:
             await datasette.ensure_permission(
@@ -563,16 +590,16 @@ class QueryView(View):
         params = {key: request.args.get(key) for key in request.args}
         sql = None
 
-        if canned_query:
-            sql = canned_query["sql"]
+        if stored_query:
+            sql = stored_query.sql
         elif "sql" in params:
             sql = params.pop("sql")
 
         # Extract any :named parameters
         named_parameters = []
-        if canned_query and canned_query.get("params"):
-            named_parameters = canned_query["params"]
-        if not named_parameters:
+        if stored_query and stored_query.parameters:
+            named_parameters = stored_query.parameters
+        if not named_parameters and sql:
             named_parameters = derive_named_parameters(sql)
         named_parameter_values = {
             named_parameter: params.get(named_parameter) or ""
@@ -597,13 +624,13 @@ class QueryView(View):
 
         params_for_query = params
 
-        if not canned_query_write:
+        if sql and not stored_query_write:
             try:
-                if not canned_query:
+                if not stored_query:
                     # For regular queries we only allow SELECT, plus other rules
                     validate_sql_select(sql)
                 else:
-                    # Canned queries can run magic parameters
+                    # Stored queries can run magic parameters
                     params_for_query = MagicParameters(sql, params, request, datasette)
                     await params_for_query.execute_params()
                 results = await datasette.execute(
@@ -613,8 +640,7 @@ class QueryView(View):
                 rows = results.rows
             except QueryInterrupted as ex:
                 raise DatasetteError(
-                    textwrap.dedent(
-                        """
+                    textwrap.dedent("""
                     <p>SQL query took too long. The time limit is controlled by the
                     <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>
                     configuration option.</p>
@@ -623,10 +649,7 @@ class QueryView(View):
                     let ta = document.querySelector("textarea");
                     ta.style.height = ta.scrollHeight + "px";
                     </script>
-                """.format(
-                            markupsafe.escape(ex.sql)
-                        )
-                    ).strip(),
+                """.format(markupsafe.escape(ex.sql))).strip(),
                     title="SQL Interrupted",
                     status=400,
                     message_is_html=True,
@@ -645,6 +668,8 @@ class QueryView(View):
 
         # Handle formats from plugins
         if format_ == "csv":
+            if not sql:
+                raise DatasetteError("?sql= is required", status=400)
 
             async def fetch_data_for_csv(request, _next=None):
                 results = await db.execute(sql, params, truncate=True)
@@ -661,7 +686,7 @@ class QueryView(View):
                 columns=columns,
                 rows=rows,
                 sql=sql,
-                query_name=canned_query["name"] if canned_query else None,
+                query_name=stored_query.name if stored_query else None,
                 database=database,
                 table=None,
                 request=request,
@@ -693,10 +718,10 @@ class QueryView(View):
         elif format_ == "html":
             headers = {}
             templates = [f"query-{to_css_class(database)}.html", "query.html"]
-            if canned_query:
+            if stored_query:
                 templates.insert(
                     0,
-                    f"query-{to_css_class(database)}-{to_css_class(canned_query['name'])}.html",
+                    f"query-{to_css_class(database)}-{to_css_class(stored_query.name)}.html",
                 )
 
             environment = datasette.get_jinja_environment(request)
@@ -714,6 +739,9 @@ class QueryView(View):
                 }
             )
             metadata = await datasette.get_database_metadata(database)
+            if stored_query:
+                metadata = stored_query_to_dict(stored_query)
+                metadata.pop("source", None)
 
             renderers = {}
             for key, (_, can_render) in datasette.renderers.items():
@@ -740,9 +768,14 @@ class QueryView(View):
                 resource=DatabaseResource(database=database),
                 actor=request.actor,
             )
+            allow_store_query = await datasette.allowed(
+                action="store-query",
+                resource=DatabaseResource(database=database),
+                actor=request.actor,
+            )
 
             show_hide_hidden = ""
-            if canned_query and canned_query.get("hide_sql"):
+            if stored_query and stored_query.hide_sql:
                 if bool(params.get("_show_sql")):
                     show_hide_link = path_with_removed_args(request, {"_show_sql"})
                     show_hide_text = "hide"
@@ -770,24 +803,38 @@ class QueryView(View):
             # - No magic parameters, so no :_ in the SQL string
             edit_sql_url = None
             is_validated_sql = False
-            try:
-                validate_sql_select(sql)
-                is_validated_sql = True
-            except InvalidSql:
-                pass
-            if allow_execute_sql and is_validated_sql and ":_" not in sql:
-                edit_sql_url = (
-                    datasette.urls.database(database)
-                    + "/-/query"
-                    + "?"
-                    + urlencode(
-                        {
-                            **{
-                                "sql": sql,
-                            },
-                            **named_parameter_values,
-                        }
+            if sql:
+                try:
+                    validate_sql_select(sql)
+                    is_validated_sql = True
+                except InvalidSql:
+                    pass
+                if allow_execute_sql and is_validated_sql and ":_" not in sql:
+                    edit_sql_url = (
+                        datasette.urls.database(database)
+                        + "/-/query"
+                        + "?"
+                        + urlencode(
+                            {
+                                **{
+                                    "sql": sql,
+                                },
+                                **named_parameter_values,
+                            }
+                        )
                     )
+            save_query_url = None
+            if (
+                not stored_query
+                and allow_execute_sql
+                and allow_store_query
+                and is_validated_sql
+                and ":_" not in sql
+            ):
+                save_query_url = (
+                    datasette.urls.database(database)
+                    + "/-/queries/store?"
+                    + urlencode({"sql": sql})
                 )
 
             async def query_actions():
@@ -796,7 +843,7 @@ class QueryView(View):
                     datasette=datasette,
                     actor=request.actor,
                     database=database,
-                    query_name=canned_query["name"] if canned_query else None,
+                    query_name=stored_query.name if stored_query else None,
                     request=request,
                     sql=sql,
                     params=params,
@@ -816,16 +863,17 @@ class QueryView(View):
                             "sql": sql,
                             "params": params,
                         },
-                        canned_query=canned_query["name"] if canned_query else None,
+                        stored_query=stored_query.name if stored_query else None,
                         private=private,
-                        canned_query_write=canned_query_write,
+                        stored_query_write=stored_query_write,
                         db_is_immutable=not db.is_mutable,
                         error=query_error,
                         hide_sql=hide_sql,
                         show_hide_link=datasette.urls.path(show_hide_link),
                         show_hide_text=show_hide_text,
-                        editable=not canned_query,
+                        editable=not stored_query,
                         allow_execute_sql=allow_execute_sql,
+                        save_query_url=save_query_url,
                         tables=await get_tables(datasette, request, db, allowed_dict),
                         named_parameter_values=named_parameter_values,
                         edit_sql_url=edit_sql_url,
@@ -845,7 +893,7 @@ class QueryView(View):
                             )
                         ),
                         show_hide_hidden=markupsafe.Markup(show_hide_hidden),
-                        metadata=canned_query or metadata,
+                        metadata=metadata,
                         alternate_url_json=alternate_url_json,
                         select_templates=[
                             f"{'*' if template_name == template.name else ''}{template_name}"
@@ -854,12 +902,12 @@ class QueryView(View):
                         top_query=make_slot_function(
                             "top_query", datasette, request, database=database, sql=sql
                         ),
-                        top_canned_query=make_slot_function(
-                            "top_canned_query",
+                        top_stored_query=make_slot_function(
+                            "top_stored_query",
                             datasette,
                             request,
                             database=database,
-                            query_name=canned_query["name"] if canned_query else None,
+                            query_name=stored_query.name if stored_query else None,
                         ),
                         query_actions=query_actions,
                     ),
@@ -1119,7 +1167,7 @@ class TableCreateView(BaseView):
             return table.schema
 
         try:
-            schema = await db.execute_write_fn(create_table)
+            schema = await db.execute_write_fn(create_table, request=request)
         except Exception as e:
             return _error([str(e)])
 
@@ -1172,22 +1220,6 @@ class TableCreateView(BaseView):
         return Response.json(details, status=201)
 
 
-async def _table_columns(datasette, database_name):
-    internal_db = datasette.get_internal_database()
-    result = await internal_db.execute(
-        "select table_name, name from catalog_columns where database_name = ?",
-        [database_name],
-    )
-    table_columns = {}
-    for row in result.rows:
-        table_columns.setdefault(row["table_name"], []).append(row["name"])
-    # Add views
-    db = datasette.get_database(database_name)
-    for view_name in await db.view_names():
-        table_columns[view_name] = []
-    return table_columns
-
-
 async def display_rows(datasette, database, request, rows, columns):
     display_rows = []
     truncate_cells = datasette.setting("truncate_cells_html")
@@ -1203,9 +1235,11 @@ async def display_rows(datasette, database, request, rows, columns):
                 value=value,
                 column=column,
                 table=None,
+                pks=[],
                 database=database,
                 datasette=datasette,
                 request=request,
+                column_type=None,
             ):
                 candidate = await await_me_maybe(candidate)
                 if candidate is not None:

datasette/views/execute_write.py

@@ -0,0 +1,257 @@
+from urllib.parse import urlencode
+
+from datasette.resources import DatabaseResource
+from datasette.utils import sqlite3
+from datasette.utils.asgi import Response
+
+from .base import BaseView, _error
+from .query_helpers import (
+    QueryValidationError,
+    _analysis_is_write,
+    _analysis_rows,
+    _analysis_rows_with_permissions,
+    _block_framing,
+    _coerce_execute_write_payload,
+    _derived_query_parameters,
+    _execute_write_analysis_data,
+    _inserted_row_url,
+    _json_or_form_payload,
+    _prepare_execute_write,
+    _table_columns,
+    _wants_json,
+)
+
+
+class ExecuteWriteView(BaseView):
+    name = "execute-write"
+    has_json_alternate = False
+
+    async def _render_form(
+        self,
+        request,
+        db,
+        *,
+        sql="",
+        parameter_values=None,
+        analysis=None,
+        analysis_error=None,
+        execution_message=None,
+        execution_links=None,
+        execution_ok=None,
+        status=200,
+    ):
+        parameter_values = parameter_values or {}
+        execution_links = execution_links or []
+        parameter_names = []
+        analysis_rows = []
+        table_columns = await _table_columns(self.ds, db.name)
+        hidden_table_names = set(await db.hidden_table_names())
+        write_template_tables = {
+            table: columns
+            for table, columns in table_columns.items()
+            if columns and table not in hidden_table_names
+        }
+        if sql and analysis_error is None:
+            try:
+                parameter_names = _derived_query_parameters(sql)
+                if analysis is None:
+                    params = {parameter: "" for parameter in parameter_names}
+                    analysis = await db.analyze_sql(sql, params)
+                if _analysis_is_write(analysis):
+                    analysis_rows = await _analysis_rows_with_permissions(
+                        self.ds, analysis, request.actor
+                    )
+                else:
+                    analysis_error = (
+                        "Use /-/query for read-only SQL; "
+                        "this endpoint only executes writes"
+                    )
+            except (QueryValidationError, sqlite3.DatabaseError) as ex:
+                analysis_error = getattr(ex, "message", str(ex))
+
+        allow_save_query = await self.ds.allowed(
+            action="execute-sql",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        ) and await self.ds.allowed(
+            action="store-query",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        )
+        save_query_base_url = None
+        save_query_url = None
+        if allow_save_query:
+            save_query_base_url = self.ds.urls.database(db.name) + "/-/queries/store"
+            if (
+                sql
+                and analysis_error is None
+                and not any(row["allowed"] is False for row in analysis_rows)
+            ):
+                save_query_url = save_query_base_url + "?" + urlencode({"sql": sql})
+
+        response = await self.render(
+            ["execute_write.html"],
+            request,
+            {
+                "database": db.name,
+                "database_color": db.color,
+                "sql": sql,
+                "parameter_names": parameter_names,
+                "parameter_values": parameter_values,
+                "analysis_error": analysis_error,
+                "analysis_rows": [
+                    row for row in analysis_rows if row["operation"] != "read"
+                ],
+                "execution_message": execution_message,
+                "execution_links": execution_links,
+                "execution_ok": execution_ok,
+                "execute_disabled": bool(
+                    (not sql)
+                    or analysis_error
+                    or any(row["allowed"] is False for row in analysis_rows)
+                ),
+                "table_columns": table_columns,
+                "write_template_tables": write_template_tables,
+                "save_query_url": save_query_url,
+                "save_query_base_url": save_query_base_url,
+            },
+        )
+        response.status = status
+        return _block_framing(response)
+
+    async def get(self, request):
+        db = await self.ds.resolve_database(request)
+        await self.ds.ensure_permission(
+            action="execute-write-sql",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        )
+        if not db.is_mutable:
+            return _block_framing(
+                _error(
+                    ["Cannot execute write SQL because this database is immutable."],
+                    403,
+                )
+            )
+        return await self._render_form(
+            request,
+            db,
+            sql=request.args.get("sql") or "",
+        )
+
+    async def post(self, request):
+        db = await self.ds.resolve_database(request)
+        if not await self.ds.allowed(
+            action="execute-write-sql",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        ):
+            return _block_framing(
+                _error(["Permission denied: need execute-write-sql"], 403)
+            )
+        if not db.is_mutable:
+            return _block_framing(_error(["Database is immutable"], 403))
+
+        data = {}
+        is_json = request.headers.get("content-type", "").startswith("application/json")
+        sql = ""
+        provided_params = {}
+        try:
+            data, is_json = await _json_or_form_payload(request)
+            sql, provided_params = _coerce_execute_write_payload(data, is_json)
+            parameter_names, params, analysis = await _prepare_execute_write(
+                self.ds, db, sql, provided_params, request.actor
+            )
+        except QueryValidationError as ex:
+            if _wants_json(request, is_json, data):
+                return _block_framing(_error([ex.message], ex.status))
+            return await self._render_form(
+                request,
+                db,
+                sql=sql or "",
+                parameter_values=provided_params,
+                analysis_error=ex.message,
+                execution_message=ex.message,
+                execution_ok=False,
+                status=ex.status,
+            )
+
+        try:
+            cursor = await db.execute_write(sql, params, request=request)
+        except sqlite3.DatabaseError as ex:
+            message = str(ex)
+            if _wants_json(request, is_json, data):
+                return _block_framing(_error([message], 400))
+            return await self._render_form(
+                request,
+                db,
+                sql=sql,
+                parameter_values=params,
+                analysis=analysis,
+                execution_message=message,
+                execution_ok=False,
+                status=400,
+            )
+
+        message = "Query executed, {} row{} affected".format(
+            cursor.rowcount, "" if cursor.rowcount == 1 else "s"
+        )
+        if _wants_json(request, is_json, data):
+            return _block_framing(
+                Response.json(
+                    {
+                        "ok": True,
+                        "message": message,
+                        "rowcount": cursor.rowcount,
+                        "analysis": _analysis_rows(analysis),
+                    }
+                )
+            )
+
+        inserted_row_url = await _inserted_row_url(self.ds, db, analysis, cursor)
+        execution_links = (
+            [{"href": inserted_row_url, "label": "View row"}]
+            if inserted_row_url
+            else []
+        )
+        return await self._render_form(
+            request,
+            db,
+            sql=sql,
+            parameter_values={name: params.get(name, "") for name in parameter_names},
+            analysis=analysis,
+            execution_message=message,
+            execution_links=execution_links,
+            execution_ok=True,
+        )
+
+
+class ExecuteWriteAnalyzeView(BaseView):
+    name = "execute-write-analyze"
+    has_json_alternate = False
+
+    async def get(self, request):
+        db = await self.ds.resolve_database(request)
+        if not await self.ds.allowed(
+            action="execute-write-sql",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        ):
+            return _block_framing(
+                _error(["Permission denied: need execute-write-sql"], 403)
+            )
+
+        invalid_keys = set(request.args) - {"sql"}
+        if invalid_keys:
+            return _block_framing(
+                _error(
+                    ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))],
+                    400,
+                )
+            )
+        sql = request.args.get("sql") or ""
+        return _block_framing(
+            Response.json(
+                await _execute_write_analysis_data(self.ds, db, sql, request.actor)
+            )
+        )

datasette/views/index.py

@@ -12,7 +12,6 @@ from datasette.version import __version__
 
 from .base import BaseView
 
-
 # Truncate table list on homepage at:
 TRUNCATE_AT = 5
 

datasette/views/query_helpers.py

@@ -0,0 +1,556 @@
+import json
+import re
+
+from datasette.resources import DatabaseResource, TableResource
+from datasette.stored_queries import StoredQuery
+from datasette.utils import (
+    named_parameters as derive_named_parameters,
+    escape_sqlite,
+    path_from_row_pks,
+    sqlite3,
+    validate_sql_select,
+    InvalidSql,
+)
+from datasette.utils.asgi import Forbidden
+
+_query_name_re = re.compile(r"^[^/\.\n]+$")
+
+_query_fields = {
+    "sql",
+    "title",
+    "description",
+    "hide_sql",
+    "fragment",
+    "parameters",
+    "params",
+    "is_private",
+    "on_success_message",
+    "on_success_redirect",
+    "on_error_message",
+    "on_error_redirect",
+}
+
+_query_create_fields = _query_fields | {"name", "mode", "csrftoken"}
+_query_update_fields = _query_fields
+_query_write_fields = {
+    "on_success_message",
+    "on_success_redirect",
+    "on_error_message",
+    "on_error_redirect",
+}
+
+
+class QueryValidationError(Exception):
+    def __init__(self, message, status=400):
+        self.message = message
+        self.status = status
+
+
+def _actor_id(actor):
+    if isinstance(actor, dict):
+        return actor.get("id")
+    return None
+
+
+def _as_bool(value):
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return False
+    if isinstance(value, int):
+        return bool(value)
+    if isinstance(value, str):
+        return value.lower() in {"1", "true", "t", "yes", "on"}
+    return bool(value)
+
+
+def _as_optional_bool(value, name):
+    if value is None or value == "":
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, int):
+        return bool(value)
+    if isinstance(value, str):
+        lowered = value.lower()
+        if lowered in {"1", "true", "t", "yes", "on"}:
+            return True
+        if lowered in {"0", "false", "f", "no", "off"}:
+            return False
+    raise QueryValidationError("{} must be 0 or 1".format(name))
+
+
+def _query_list_limit(value, default=50):
+    if value in (None, ""):
+        return default
+    try:
+        return min(max(1, int(value)), 1000)
+    except ValueError as ex:
+        raise QueryValidationError("_size must be an integer") from ex
+
+
+def _derived_query_parameters(sql):
+    parameters = []
+    seen = set()
+    for parameter in derive_named_parameters(sql):
+        if parameter.startswith("_"):
+            raise QueryValidationError("Magic parameters are not allowed")
+        if parameter not in seen:
+            parameters.append(parameter)
+            seen.add(parameter)
+    return parameters
+
+
+def _coerce_query_parameters(value, derived):
+    if value is None:
+        return derived
+    if isinstance(value, str):
+        parameters = [
+            parameter.strip()
+            for parameter in re.split(r"[\s,]+", value)
+            if parameter.strip()
+        ]
+    elif isinstance(value, list):
+        parameters = value
+    else:
+        raise QueryValidationError("parameters must be a list of strings")
+    if not all(isinstance(parameter, str) for parameter in parameters):
+        raise QueryValidationError("parameters must be a list of strings")
+    if any(parameter.startswith("_") for parameter in parameters):
+        raise QueryValidationError("Magic parameters are not allowed")
+    if set(parameters) != set(derived):
+        raise QueryValidationError("parameters must match SQL named parameters")
+    return parameters
+
+
+def _analysis_is_write(analysis):
+    return any(
+        access.operation in {"insert", "update", "delete"}
+        for access in analysis.table_accesses
+    )
+
+
+def _block_framing(response):
+    response.headers["Content-Security-Policy"] = "frame-ancestors 'none'"
+    response.headers["X-Frame-Options"] = "DENY"
+    return response
+
+
+def _wants_json(request, is_json, data):
+    return (
+        is_json
+        or request.headers.get("accept") == "application/json"
+        or (isinstance(data, dict) and data.get("_json"))
+    )
+
+
+def _query_create_form_error_message(message):
+    return {
+        "Query name is required": "URL is required",
+        "Invalid query name": "Invalid URL",
+        "Query name conflicts with a table or view": (
+            "URL conflicts with an existing table or view"
+        ),
+        "Query already exists": "A query already exists at that URL",
+    }.get(message, message)
+
+
+async def _json_or_form_payload(request):
+    content_type = request.headers.get("content-type", "")
+    if content_type.startswith("application/json"):
+        body = await request.post_body()
+        try:
+            return json.loads(body or b"{}"), True
+        except json.JSONDecodeError as e:
+            raise QueryValidationError("Invalid JSON: {}".format(e))
+    return await request.post_vars(), False
+
+
+async def _check_query_name(db, name, *, existing=False):
+    if not name or not isinstance(name, str):
+        raise QueryValidationError("Query name is required")
+    if not _query_name_re.match(name):
+        raise QueryValidationError("Invalid query name")
+    if not existing and (await db.table_exists(name) or await db.view_exists(name)):
+        raise QueryValidationError("Query name conflicts with a table or view")
+
+
+async def _analyze_user_query(datasette, db, sql, *, actor):
+    if not sql or not isinstance(sql, str):
+        raise QueryValidationError("SQL is required")
+    derived = _derived_query_parameters(sql)
+    params = {parameter: "" for parameter in derived}
+    try:
+        analysis = await db.analyze_sql(sql, params)
+    except sqlite3.DatabaseError as ex:
+        raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex
+
+    is_write = _analysis_is_write(analysis)
+    if is_write:
+        try:
+            await datasette.ensure_query_write_permissions(
+                db.name, sql, actor=actor, analysis=analysis
+            )
+        except Forbidden as ex:
+            raise QueryValidationError(str(ex), status=403) from ex
+    else:
+        try:
+            validate_sql_select(sql)
+        except InvalidSql as ex:
+            raise QueryValidationError(str(ex)) from ex
+    return is_write, derived, analysis
+
+
+def _analysis_rows(analysis):
+    write_actions = {
+        "insert": "insert-row",
+        "update": "update-row",
+        "delete": "delete-row",
+    }
+    return [
+        {
+            "operation": access.operation,
+            "database": access.database,
+            "table": access.table,
+            "required_permission": write_actions.get(access.operation, ""),
+            "source": access.source,
+        }
+        for access in analysis.table_accesses
+    ]
+
+
+async def _analysis_rows_with_permissions(datasette, analysis, actor):
+    rows = _analysis_rows(analysis)
+    for row in rows:
+        permission = row["required_permission"]
+        if permission:
+            row["allowed"] = await datasette.allowed(
+                action=permission,
+                resource=TableResource(row["database"], row["table"]),
+                actor=actor,
+            )
+        else:
+            row["allowed"] = None
+    return rows
+
+
+def _coerce_execute_write_payload(data, is_json):
+    if not isinstance(data, dict):
+        raise QueryValidationError("JSON must be a dictionary")
+    if is_json:
+        invalid_keys = set(data) - {"sql", "params"}
+        if invalid_keys:
+            raise QueryValidationError(
+                "Invalid keys: {}".format(", ".join(sorted(invalid_keys)))
+            )
+        params = data.get("params") or {}
+    else:
+        params = {
+            key: value
+            for key, value in data.items()
+            if key not in {"sql", "csrftoken", "_json"}
+        }
+    if not isinstance(params, dict):
+        raise QueryValidationError("params must be a dictionary")
+    return data.get("sql"), params
+
+
+async def _prepare_execute_write(datasette, db, sql, params, actor):
+    if not sql or not isinstance(sql, str):
+        raise QueryValidationError("SQL is required")
+    parameter_names = _derived_query_parameters(sql)
+    extra_params = set(params) - set(parameter_names)
+    if extra_params:
+        raise QueryValidationError(
+            "Unknown parameters: {}".format(", ".join(sorted(extra_params)))
+        )
+    params = {name: params.get(name, "") for name in parameter_names}
+    try:
+        analysis = await db.analyze_sql(sql, params)
+    except sqlite3.DatabaseError as ex:
+        raise QueryValidationError("Could not analyze query: {}".format(ex)) from ex
+    if not _analysis_is_write(analysis):
+        raise QueryValidationError(
+            "Use /-/query for read-only SQL; this endpoint only executes writes"
+        )
+    try:
+        await datasette.ensure_query_write_permissions(
+            db.name, sql, actor=actor, analysis=analysis
+        )
+    except Forbidden as ex:
+        raise QueryValidationError(str(ex), status=403) from ex
+    return parameter_names, params, analysis
+
+
+async def _ensure_stored_query_execution_permissions(
+    datasette, db, query: StoredQuery, actor
+):
+    if query.is_trusted:
+        return
+    if query.is_write:
+        await datasette.ensure_permission(
+            action="execute-write-sql",
+            resource=DatabaseResource(db.name),
+            actor=actor,
+        )
+        await datasette.ensure_query_write_permissions(db.name, query.sql, actor=actor)
+    else:
+        await datasette.ensure_permission(
+            action="execute-sql",
+            resource=DatabaseResource(db.name),
+            actor=actor,
+        )
+
+
+async def _execute_write_analysis_data(datasette, db, sql, actor):
+    parameter_names = []
+    analysis_rows = []
+    analysis_error = None
+    if sql:
+        try:
+            parameter_names = _derived_query_parameters(sql)
+            params = {parameter: "" for parameter in parameter_names}
+            analysis = await db.analyze_sql(sql, params)
+            if _analysis_is_write(analysis):
+                analysis_rows = await _analysis_rows_with_permissions(
+                    datasette, analysis, actor
+                )
+            else:
+                analysis_error = (
+                    "Use /-/query for read-only SQL; "
+                    "this endpoint only executes writes"
+                )
+        except (QueryValidationError, sqlite3.DatabaseError) as ex:
+            analysis_error = getattr(ex, "message", str(ex))
+    return {
+        "ok": analysis_error is None,
+        "parameters": parameter_names,
+        "analysis_error": analysis_error,
+        "analysis_rows": [row for row in analysis_rows if row["operation"] != "read"],
+        "execute_disabled": bool(
+            (not sql)
+            or analysis_error
+            or any(row["allowed"] is False for row in analysis_rows)
+        ),
+    }
+
+
+async def _query_create_analysis_data(datasette, db, sql, actor):
+    has_sql = bool(sql and sql.strip())
+    parameter_names = []
+    analysis_rows = []
+    analysis_error = None
+    if has_sql:
+        try:
+            parameter_names = _derived_query_parameters(sql)
+            params = {parameter: "" for parameter in parameter_names}
+            analysis = await db.analyze_sql(sql, params)
+            analysis_rows = await _analysis_rows_with_permissions(
+                datasette, analysis, actor
+            )
+        except (QueryValidationError, sqlite3.DatabaseError) as ex:
+            analysis_error = getattr(ex, "message", str(ex))
+    return {
+        "ok": analysis_error is None,
+        "parameters": parameter_names,
+        "analysis_error": analysis_error,
+        "analysis_rows": analysis_rows,
+        "has_sql": has_sql,
+        "analysis_is_write": bool(
+            analysis_rows and any(row["required_permission"] for row in analysis_rows)
+        ),
+        "save_disabled": bool(
+            (not has_sql)
+            or analysis_error
+            or any(row["allowed"] is False for row in analysis_rows)
+        ),
+    }
+
+
+async def _query_create_form_context(
+    datasette,
+    request,
+    db,
+    *,
+    sql="",
+    name="",
+    title="",
+    description="",
+    is_private=True,
+):
+    analysis_data = await _query_create_analysis_data(datasette, db, sql, request.actor)
+    return {
+        "database": db.name,
+        "database_color": db.color,
+        "sql": sql,
+        "name": name,
+        "title": title,
+        "description": description,
+        "is_private": is_private,
+        **analysis_data,
+    }
+
+
+async def _inserted_row_url(datasette, db, analysis, cursor):
+    if cursor.rowcount != 1:
+        return None
+    lastrowid = getattr(cursor, "lastrowid", None)
+    if lastrowid is None:
+        return None
+    direct_inserts = [
+        access
+        for access in analysis.table_accesses
+        if access.operation == "insert"
+        and access.source is None
+        and access.database == db.name
+    ]
+    if len(direct_inserts) != 1:
+        return None
+    table = direct_inserts[0].table
+    pks = await db.primary_keys(table)
+    use_rowid = not pks
+    select = (
+        "rowid"
+        if use_rowid
+        else ", ".join(escape_sqlite(primary_key) for primary_key in pks)
+    )
+    try:
+        result = await db.execute(
+            "select {} from {} where rowid = ?".format(select, escape_sqlite(table)),
+            [lastrowid],
+        )
+    except sqlite3.DatabaseError:
+        return None
+    row = result.first()
+    if row is None:
+        return None
+    row_path = path_from_row_pks(row, pks, use_rowid)
+    return datasette.urls.row(db.name, table, row_path)
+
+
+def _apply_query_data_types(data):
+    typed = dict(data)
+    for key in ("hide_sql", "is_private"):
+        if key in typed:
+            typed[key] = _as_bool(typed[key])
+    return typed
+
+
+async def _prepare_query_create(datasette, request, db, data):
+    invalid_keys = set(data) - _query_create_fields
+    if invalid_keys:
+        raise QueryValidationError(
+            "Invalid keys: {}".format(", ".join(sorted(invalid_keys)))
+        )
+
+    data = _apply_query_data_types(data)
+    name = data.get("name")
+    await _check_query_name(db, name)
+    if await datasette.get_query(db.name, name) is not None:
+        raise QueryValidationError("Query already exists")
+
+    is_write, derived, analysis = await _analyze_user_query(
+        datasette,
+        db,
+        data.get("sql"),
+        actor=request.actor,
+    )
+    if not is_write and any(data.get(field) for field in _query_write_fields):
+        raise QueryValidationError("Writable query fields require writable SQL")
+
+    parameters = _coerce_query_parameters(
+        data.get("parameters", data.get("params")),
+        derived,
+    )
+    return {
+        "name": name,
+        "sql": data["sql"],
+        "title": data.get("title"),
+        "description": data.get("description"),
+        "hide_sql": _as_bool(data.get("hide_sql")),
+        "fragment": data.get("fragment"),
+        "parameters": parameters,
+        "is_write": is_write,
+        "is_private": _as_bool(data.get("is_private", True)),
+        "is_trusted": False,
+        "source": "user",
+        "owner_id": _actor_id(request.actor),
+        "on_success_message": data.get("on_success_message"),
+        "on_success_redirect": data.get("on_success_redirect"),
+        "on_error_message": data.get("on_error_message"),
+        "on_error_redirect": data.get("on_error_redirect"),
+        "analysis": analysis,
+    }
+
+
+async def _prepare_query_update(datasette, request, db, existing: StoredQuery, update):
+    invalid_keys = set(update) - _query_update_fields
+    if invalid_keys:
+        raise QueryValidationError(
+            "Invalid keys: {}".format(", ".join(sorted(invalid_keys)))
+        )
+
+    update = _apply_query_data_types(update)
+    sql = update.get("sql", existing.sql)
+    query_is_write = existing.is_write
+    derived = _derived_query_parameters(sql)
+    parameters = None
+
+    if "sql" in update:
+        query_is_write, derived, _ = await _analyze_user_query(
+            datasette,
+            db,
+            sql,
+            actor=request.actor,
+        )
+
+    if "parameters" in update or "params" in update:
+        parameters = _coerce_query_parameters(
+            update.get("parameters", update.get("params")),
+            derived,
+        )
+    elif "sql" in update:
+        parameters = derived
+
+    if not query_is_write and any(update.get(field) for field in _query_write_fields):
+        raise QueryValidationError("Writable query fields require writable SQL")
+
+    field_values = {
+        "sql": sql,
+        "title": update.get("title"),
+        "description": update.get("description"),
+        "hide_sql": update.get("hide_sql"),
+        "fragment": update.get("fragment"),
+        "parameters": parameters,
+        "is_write": query_is_write,
+        "is_private": update.get("is_private"),
+        "on_success_message": update.get("on_success_message"),
+        "on_success_redirect": update.get("on_success_redirect"),
+        "on_error_message": update.get("on_error_message"),
+        "on_error_redirect": update.get("on_error_redirect"),
+    }
+    update_kwargs = {}
+    for field_name, value in field_values.items():
+        if field_name in update:
+            update_kwargs[field_name] = value
+    if parameters is not None:
+        update_kwargs["parameters"] = parameters
+    if "sql" in update:
+        update_kwargs["is_write"] = query_is_write
+    return update_kwargs
+
+
+async def _table_columns(datasette, database_name):
+    internal_db = datasette.get_internal_database()
+    result = await internal_db.execute(
+        "select table_name, name from catalog_columns where database_name = ?",
+        [database_name],
+    )
+    table_columns = {}
+    for row in result.rows:
+        table_columns.setdefault(row["table_name"], []).append(row["name"])
+    # Add views
+    db = datasette.get_database(database_name)
+    for view_name in await db.view_names():
+        table_columns[view_name] = []
+    return table_columns

datasette/views/row.py

@@ -5,14 +5,16 @@ from datasette.resources import TableResource
 from .base import DataView, BaseView, _error
 from datasette.utils import (
     await_me_maybe,
+    CustomRow,
     make_slot_function,
     to_css_class,
     escape_sqlite,
 )
 from datasette.plugins import pm
 import json
+import markupsafe
 import sqlite_utils
-from .table import display_columns_and_rows
+from .table import display_columns_and_rows, _get_extras
 
 
 class RowView(DataView):
@@ -42,13 +44,62 @@ class RowView(DataView):
         if not rows:
             raise NotFound(f"Record not found: {pk_values}")
 
+        pks = resolved.pks
+
         async def template_data():
+            # Reorder columns so primary keys come first
+            pk_set = set(pks)
+            pk_cols = [d for d in results.description if d[0] in pk_set]
+            non_pk_cols = [d for d in results.description if d[0] not in pk_set]
+            reordered_description = pk_cols + non_pk_cols
+            reordered_columns = [d[0] for d in reordered_description]
+
+            # Reorder row data to match
+            reordered_rows = []
+            for row in rows:
+                new_row = CustomRow(reordered_columns)
+                for col in reordered_columns:
+                    new_row[col] = row[col]
+                reordered_rows.append(new_row)
+
+            # Expand foreign key columns into dicts so display_columns_and_rows
+            # renders them as hyperlinks, matching the table view behavior
+            expanded_rows = reordered_rows
+            for fk in await db.foreign_keys_for_table(table):
+                column = fk["column"]
+                if column not in reordered_columns:
+                    continue
+                column_index = reordered_columns.index(column)
+                values = [row[column_index] for row in expanded_rows]
+                expanded_labels = await self.ds.expand_foreign_keys(
+                    request.actor, database, table, column, values
+                )
+                if expanded_labels:
+                    new_rows = []
+                    for row in expanded_rows:
+                        new_row = CustomRow(reordered_columns)
+                        for col in reordered_columns:
+                            value = row[col]
+                            if (
+                                col == column
+                                and (col, value) in expanded_labels
+                                and value is not None
+                            ):
+                                new_row[col] = {
+                                    "value": value,
+                                    "label": expanded_labels[(col, value)],
+                                }
+                            else:
+                                new_row[col] = value
+                        new_rows.append(new_row)
+                    expanded_rows = new_rows
+
             display_columns, display_rows = await display_columns_and_rows(
                 self.ds,
                 database,
                 table,
-                results.description,
-                rows,
+                reordered_description,
+                expanded_rows,
                 link_column=False,
                 truncate_cells=0,
                 request=request,
@@ -56,6 +107,14 @@ class RowView(DataView):
             for column in display_columns:
                 column["sortable"] = False
 
+            # Bold primary key cell values
+            for row in display_rows:
+                for cell in row:
+                    if cell["column"] in pk_set:
+                        cell["value"] = markupsafe.Markup(
+                            "<strong>{}</strong>".format(cell["value"])
+                        )
+
             row_actions = []
             for hook in pm.hook.row_actions(
                 datasette=self.ds,
@@ -71,6 +130,7 @@ class RowView(DataView):
 
             return {
                 "private": private,
+                "columns": reordered_columns,
                 "foreign_key_tables": await self.foreign_key_tables(
                     database, table, pk_values
                 ),
@@ -95,6 +155,7 @@ class RowView(DataView):
             }
 
         data = {
+            "ok": True,
             "database": database,
             "table": table,
             "rows": rows,
@@ -103,11 +164,61 @@ class RowView(DataView):
             "primary_key_values": pk_values,
         }
 
+        # Handle _extra parameter (new style)
+        extras = _get_extras(request)
+
+        # Also support legacy _extras parameter for backward compatibility
         if "foreign_key_tables" in (request.args.get("_extras") or "").split(","):
+            extras.add("foreign_key_tables")
+
+        # Process extras
+        if "foreign_key_tables" in extras:
             data["foreign_key_tables"] = await self.foreign_key_tables(
                 database, table, pk_values
             )
 
+        if "render_cell" in extras:
+            # Call render_cell plugin hook for each cell
+            ct_map = await self.ds.get_column_types(database, table)
+            rendered_rows = []
+            for row in rows:
+                rendered_row = {}
+                for value, column in zip(row, columns):
+                    ct = ct_map.get(column)
+                    plugin_display_value = None
+                    # Try column type render_cell first
+                    if ct:
+                        candidate = await ct.render_cell(
+                            value=value,
+                            column=column,
+                            table=table,
+                            database=database,
+                            datasette=self.ds,
+                            request=request,
+                        )
+                        if candidate is not None:
+                            plugin_display_value = candidate
+                    if plugin_display_value is None:
+                        for candidate in pm.hook.render_cell(
+                            row=row,
+                            value=value,
+                            column=column,
+                            table=table,
+                            pks=resolved.pks,
+                            database=database,
+                            datasette=self.ds,
+                            request=request,
+                            column_type=ct,
+                        ):
+                            candidate = await await_me_maybe(candidate)
+                            if candidate is not None:
+                                plugin_display_value = candidate
+                                break
+                    if plugin_display_value:
+                        rendered_row[column] = str(plugin_display_value)
+                rendered_rows.append(rendered_row)
+            data["render_cell"] = rendered_rows
+
         return (
             data,
             template_data,
@@ -210,7 +321,7 @@ class RowDeleteView(BaseView):
             sqlite_utils.Database(conn)[resolved.table].delete(resolved.pk_values)
 
         try:
-            await resolved.db.execute_write_fn(delete_row)
+            await resolved.db.execute_write_fn(delete_row, request=request)
         except Exception as e:
             return _error([str(e)], 500)
 
@@ -256,6 +367,15 @@ class RowUpdateView(BaseView):
 
         update = data["update"]
 
+        # Validate column types
+        from datasette.views.table import _validate_column_types
+
+        ct_errors = await _validate_column_types(
+            self.ds, resolved.db.name, resolved.table, [update]
+        )
+        if ct_errors:
+            return _error(ct_errors, 400)
+
         alter = data.get("alter")
         if alter and not await self.ds.allowed(
             action="alter-table",
@@ -270,7 +390,7 @@ class RowUpdateView(BaseView):
             )
 
         try:
-            await resolved.db.execute_write_fn(update_row)
+            await resolved.db.execute_write_fn(update_row, request=request)
         except Exception as e:
             return _error([str(e)], 400)
 

datasette/views/special.py

@@ -1,11 +1,14 @@
 import json
 import logging
+from datasette.jump import JumpSQL, namespace_sql_params
+from datasette.plugins import pm
 from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent
 from datasette.resources import DatabaseResource, TableResource
 from datasette.utils.asgi import Response, Forbidden
 from datasette.utils import (
     actor_matches_allow,
     add_cors_headers,
+    await_me_maybe,
     tilde_encode,
     tilde_decode,
 )
@@ -13,7 +16,6 @@ from .base import BaseView, View
 import secrets
 import urllib
 
-
 logger = logging.getLogger(__name__)
 
 
@@ -65,7 +67,7 @@ class JsonDataView(BaseView):
             context = {
                 "filename": self.filename,
                 "data": data,
-                "data_json": json.dumps(data, indent=4, default=repr),
+                "data_json": json.dumps(data, indent=2, default=repr),
             }
             # Add has_debug_permission if this view requires permissions-debug
             if self.permission == "permissions-debug":
@@ -177,11 +179,11 @@ class PermissionsDebugView(BaseView):
     async def post(self, request):
         await self.ds.ensure_permission(action="view-instance", actor=request.actor)
         await self.ds.ensure_permission(action="permissions-debug", actor=request.actor)
-        vars = await request.post_vars()
-        actor = json.loads(vars["actor"])
-        permission = vars["permission"]
-        parent = vars.get("resource_1") or None
-        child = vars.get("resource_2") or None
+        form = await request.form()
+        actor = json.loads(form["actor"])
+        permission = form["permission"]
+        parent = form.get("resource_1") or None
+        child = form.get("resource_2") or None
 
         response, status = await _check_permission_for_actor(
             self.ds, permission, parent, child, actor
@@ -403,7 +405,7 @@ class PermissionRulesView(BaseView):
 
         from datasette.utils.actions_sql import build_permission_rules_sql
 
-        union_sql, union_params = await build_permission_rules_sql(
+        union_sql, union_params, restriction_sqls = await build_permission_rules_sql(
             self.ds, actor, action
         )
         await self.ds.refresh_schemas()
@@ -602,9 +604,9 @@ class MessagesDebugView(BaseView):
 
     async def post(self, request):
         await self.ds.ensure_permission(action="view-instance", actor=request.actor)
-        post = await request.post_vars()
-        message = post.get("message", "")
-        message_type = post.get("message_type") or "INFO"
+        form = await request.form()
+        message = form.get("message", "")
+        message_type = form.get("message_type") or "INFO"
         assert message_type in ("INFO", "WARNING", "ERROR", "all")
         datasette = self.ds
         if message_type == "all":
@@ -688,11 +690,11 @@ class CreateTokenView(BaseView):
 
     async def post(self, request):
         self.check_permission(request)
-        post = await request.post_vars()
+        form = await request.form()
         errors = []
         expires_after = None
-        if post.get("expire_type"):
-            duration_string = post.get("expire_duration")
+        if form.get("expire_type"):
+            duration_string = form.get("expire_duration")
             if (
                 not duration_string
                 or not duration_string.isdigit()
@@ -700,7 +702,7 @@ class CreateTokenView(BaseView):
             ):
                 errors.append("Invalid expire duration")
             else:
-                unit = post["expire_type"]
+                unit = form["expire_type"]
                 if unit == "minutes":
                     expires_after = int(duration_string) * 60
                 elif unit == "hours":
@@ -711,42 +713,36 @@ class CreateTokenView(BaseView):
                     errors.append("Invalid expire duration unit")
 
         # Are there any restrictions?
-        restrict_all = []
-        restrict_database = {}
-        restrict_resource = {}
+        from datasette.tokens import TokenRestrictions
 
-        for key in post:
+        restrictions = TokenRestrictions()
+
+        for key in form:
             if key.startswith("all:") and key.count(":") == 1:
-                restrict_all.append(key.split(":")[1])
+                restrictions.allow_all(key.split(":")[1])
             elif key.startswith("database:") and key.count(":") == 2:
                 bits = key.split(":")
-                database = tilde_decode(bits[1])
-                action = bits[2]
-                restrict_database.setdefault(database, []).append(action)
+                restrictions.allow_database(tilde_decode(bits[1]), bits[2])
             elif key.startswith("resource:") and key.count(":") == 3:
                 bits = key.split(":")
-                database = tilde_decode(bits[1])
-                resource = tilde_decode(bits[2])
-                action = bits[3]
-                restrict_resource.setdefault(database, {}).setdefault(
-                    resource, []
-                ).append(action)
+                restrictions.allow_resource(
+                    tilde_decode(bits[1]), tilde_decode(bits[2]), bits[3]
+                )
 
-        token = self.ds.create_token(
+        token = await self.ds.create_token(
             request.actor["id"],
             expires_after=expires_after,
-            restrict_all=restrict_all,
-            restrict_database=restrict_database,
-            restrict_resource=restrict_resource,
+            restrictions=restrictions,
+            handler="signed",
         )
         token_bits = self.ds.unsign(token[len("dstok_") :], namespace="token")
         await self.ds.track_event(
             CreateTokenEvent(
                 actor=request.actor,
                 expires_after=expires_after,
-                restrict_all=restrict_all,
-                restrict_database=restrict_database,
-                restrict_resource=restrict_resource,
+                restrict_all=restrictions.all,
+                restrict_database=restrictions.database,
+                restrict_resource=restrictions.resource,
             )
         )
         context = await self.shared(request)
@@ -761,8 +757,6 @@ class ApiExplorerView(BaseView):
     async def example_links(self, request):
         databases = []
         for name, db in self.ds.databases.items():
-            if name == "_internal":
-                continue
             database_visible, _ = await self.ds.check_visibility(
                 request.actor,
                 action="view-database",
@@ -822,9 +816,18 @@ class ApiExplorerView(BaseView):
                                 "json": {
                                     "rows": [
                                         {
-                                            column: None
-                                            for column in await db.table_columns(table)
-                                            if column not in pks
+                                            column: "<{}{}>".format(
+                                                column,
+                                                (
+                                                    " (primary key)"
+                                                    if column in (pks or ["rowid"])
+                                                    else ""
+                                                ),
+                                            )
+                                            for column in (
+                                                (["rowid"] if not pks else [])
+                                                + await db.table_columns(table)
+                                            )
                                         }
                                     ]
                                 },
@@ -910,74 +913,359 @@ class ApiExplorerView(BaseView):
         )
 
 
-class TablesView(BaseView):
+class JumpView(BaseView):
     """
-    Simple endpoint that uses the new allowed_resources() API.
-    Returns JSON list of all tables the actor can view.
-
-    Supports ?q=foo+bar to filter tables matching .*foo.*bar.* pattern,
-    ordered by shortest name first.
+    Endpoint for the jump menu. Returns JSON navigation items the actor can use.
     """
 
-    name = "tables"
+    name = "jump"
     has_json_alternate = False
 
-    async def get(self, request):
-        # Get search query parameter
-        q = request.args.get("q", "").strip()
+    async def _fragments(self, request):
+        fragments = []
+        for hook in pm.hook.jump_items_sql(
+            datasette=self.ds,
+            actor=request.actor,
+            request=request,
+        ):
+            value = await await_me_maybe(hook)
+            if value is None:
+                continue
+            if isinstance(value, JumpSQL):
+                fragments.append(value)
+            elif isinstance(value, (list, tuple)):
+                for fragment in value:
+                    if fragment is not None:
+                        assert isinstance(
+                            fragment, JumpSQL
+                        ), "jump_items_sql must return JumpSQL instances"
+                        fragments.append(fragment)
+            else:
+                raise TypeError("jump_items_sql must return JumpSQL instances")
+        return fragments
 
-        # Get SQL for allowed resources using the permission system
-        permission_sql, params = await self.ds.allowed_resources_sql(
-            action="view-table", actor=request.actor
-        )
+    def _resolve_url(self, url):
+        if not url or url.startswith("/"):
+            return url
 
-        # Build query based on whether we have a search query
-        if q:
-            # Build SQL LIKE pattern from search terms
-            # Split search terms by whitespace and build pattern: %term1%term2%term3%
-            terms = q.split()
-            pattern = "%" + "%".join(terms) + "%"
+        descriptor = json.loads(url)
+        if not isinstance(descriptor, dict):
+            raise TypeError("jump item url JSON must be an object")
+        method_name = descriptor.get("method")
+        if not isinstance(method_name, str) or not method_name:
+            raise TypeError("jump item url JSON must include a method")
+        if method_name.startswith("_"):
+            raise AttributeError(f"datasette.urls has no method named {method_name!r}")
+        try:
+            method = getattr(self.ds.urls, method_name)
+        except AttributeError as ex:
+            raise AttributeError(
+                f"datasette.urls has no method named {method_name!r}"
+            ) from ex
+        if not callable(method):
+            raise TypeError(f"datasette.urls.{method_name} is not callable")
+        kwargs = {key: value for key, value in descriptor.items() if key != "method"}
+        try:
+            return method(**kwargs)
+        except TypeError as ex:
+            raise TypeError(
+                f"Invalid arguments for datasette.urls.{method_name}(): {ex}"
+            ) from ex
 
-            # Build query with CTE to filter by search pattern
-            sql = f"""
-            WITH allowed_tables AS (
-                {permission_sql}
-            )
-            SELECT parent, child
-            FROM allowed_tables
-            WHERE child LIKE :pattern COLLATE NOCASE
-            ORDER BY length(child), child
-            """
-            all_params = {**params, "pattern": pattern}
+    def _sort_key(self, row, q):
+        display_label = row["display_name"] or row["label"]
+        display_label_lower = display_label.lower()
+        q_lower = q.lower()
+        if display_label_lower == q_lower:
+            relevance = 0
+        elif display_label_lower.startswith(q_lower):
+            relevance = 1
         else:
-            # No search query - return all tables, ordered by name
-            # Fetch 101 to detect if we need to truncate
-            sql = f"""
-            WITH allowed_tables AS (
-                {permission_sql}
+            relevance = 2
+        type_sort = {
+            "database": 10,
+            "table": 20,
+            "view": 25,
+            "query": 30,
+        }.get(row["type"], 50)
+        return (relevance, type_sort, len(display_label), row["label"])
+
+    async def _rows_for_database(self, database_name, indexed_fragments, q, pattern):
+        params = {"q": q, "pattern": pattern}
+        union_parts = []
+        for index, fragment in indexed_fragments:
+            fragment_sql, fragment_params = namespace_sql_params(
+                fragment.sql,
+                fragment.params or {},
+                f"jump_{index}",
             )
-            SELECT parent, child
-            FROM allowed_tables
-            ORDER BY parent, child
-            LIMIT 101
-            """
-            all_params = params
+            union_parts.append(f"""
+                SELECT
+                    type,
+                    label,
+                    description,
+                    url,
+                    search_text,
+                    display_name
+                FROM (
+                    {fragment_sql}
+                )
+            """)
+            params.update(fragment_params)
+        sql = f"""
+        WITH jump_items AS (
+            {" UNION ALL ".join(union_parts)}
+        )
+        SELECT
+            type,
+            label,
+            description,
+            url,
+            search_text,
+            display_name
+        FROM jump_items
+        WHERE :q = ''
+           OR search_text LIKE :pattern COLLATE NOCASE
+        ORDER BY
+            CASE
+                WHEN lower(COALESCE(display_name, label)) = lower(:q) THEN 0
+                WHEN lower(COALESCE(display_name, label)) LIKE lower(:q || '%') THEN 1
+                ELSE 2
+            END,
+            CASE type
+                WHEN 'database' THEN 10
+                WHEN 'table' THEN 20
+                WHEN 'view' THEN 25
+                WHEN 'query' THEN 30
+                ELSE 50
+            END,
+            length(COALESCE(display_name, label)),
+            label
+        LIMIT 101
+        """
+        db = (
+            self.ds.get_internal_database()
+            if database_name is None
+            else self.ds.get_database(database_name)
+        )
+        result = await db.execute(sql, params)
+        return list(result.rows)
 
-        # Execute against internal database
-        result = await self.ds.get_internal_database().execute(sql, all_params)
+    async def get(self, request):
+        q = request.args.get("q", "").strip()
+        terms = q.split()
+        pattern = "%" + "%".join(terms) + "%" if terms else "%"
+        fragments = await self._fragments(request)
 
-        # Build response with truncation
-        rows = list(result.rows)
-        truncated = len(rows) > 100
-        if truncated:
+        fragments_by_database = {}
+        for index, fragment in enumerate(fragments):
+            fragments_by_database.setdefault(fragment.database, []).append(
+                (index, fragment)
+            )
+
+        rows = []
+        truncated = False
+        for database_name, indexed_fragments in fragments_by_database.items():
+            database_rows = await self._rows_for_database(
+                database_name, indexed_fragments, q, pattern
+            )
+            if len(database_rows) > 100:
+                truncated = True
+                database_rows = database_rows[:100]
+            rows.extend(database_rows)
+        rows.sort(key=lambda row: self._sort_key(row, q))
+
+        if len(rows) > 100:
+            truncated = True
             rows = rows[:100]
 
-        matches = [
-            {
-                "name": f"{row['parent']}: {row['child']}",
-                "url": self.ds.urls.table(row["parent"], row["child"]),
+        matches = []
+        for row in rows:
+            match = {
+                "name": row["label"],
+                "url": self._resolve_url(row["url"]),
+                "type": row["type"],
+                "description": row["description"],
             }
-            for row in rows
-        ]
+            if row["display_name"]:
+                match["display_name"] = row["display_name"]
+            matches.append(match)
 
         return Response.json({"matches": matches, "truncated": truncated})
+
+
+class SchemaBaseView(BaseView):
+    """Base class for schema views with common response formatting."""
+
+    has_json_alternate = False
+
+    async def get_database_schema(self, database_name):
+        """Get schema SQL for a database."""
+        db = self.ds.databases[database_name]
+        result = await db.execute(
+            "select group_concat(sql, ';' || CHAR(10)) as schema from sqlite_master where sql is not null"
+        )
+        row = result.first()
+        return row["schema"] if row and row["schema"] else ""
+
+    def format_json_response(self, data):
+        """Format data as JSON response with CORS headers if needed."""
+        headers = {}
+        if self.ds.cors:
+            add_cors_headers(headers)
+        return Response.json(data, headers=headers)
+
+    def format_error_response(self, error_message, format_, status=404):
+        """Format error response based on requested format."""
+        if format_ == "json":
+            headers = {}
+            if self.ds.cors:
+                add_cors_headers(headers)
+            return Response.json(
+                {"ok": False, "error": error_message}, status=status, headers=headers
+            )
+        else:
+            return Response.text(error_message, status=status)
+
+    def format_markdown_response(self, heading, schema):
+        """Format schema as Markdown response."""
+        md_output = f"# {heading}\n\n```sql\n{schema}\n```\n"
+        return Response.text(
+            md_output, headers={"content-type": "text/markdown; charset=utf-8"}
+        )
+
+    async def format_html_response(
+        self, request, schemas, is_instance=False, table_name=None
+    ):
+        """Format schema as HTML response."""
+        context = {
+            "schemas": schemas,
+            "is_instance": is_instance,
+        }
+        if table_name:
+            context["table_name"] = table_name
+        return await self.render(["schema.html"], request=request, context=context)
+
+
+class InstanceSchemaView(SchemaBaseView):
+    """
+    Displays schema for all databases in the instance.
+    Supports HTML, JSON, and Markdown formats.
+    """
+
+    name = "instance_schema"
+
+    async def get(self, request):
+        format_ = request.url_vars.get("format") or "html"
+
+        # Get all databases the actor can view
+        allowed_databases_page = await self.ds.allowed_resources(
+            "view-database",
+            request.actor,
+        )
+        allowed_databases = [r.parent async for r in allowed_databases_page.all()]
+
+        # Get schema for each database
+        schemas = []
+        for database_name in allowed_databases:
+            schema = await self.get_database_schema(database_name)
+            schemas.append({"database": database_name, "schema": schema})
+
+        if format_ == "json":
+            return self.format_json_response({"schemas": schemas})
+        elif format_ == "md":
+            md_parts = [
+                f"# Schema for {item['database']}\n\n```sql\n{item['schema']}\n```"
+                for item in schemas
+            ]
+            return Response.text(
+                "\n\n".join(md_parts),
+                headers={"content-type": "text/markdown; charset=utf-8"},
+            )
+        else:
+            return await self.format_html_response(request, schemas, is_instance=True)
+
+
+class DatabaseSchemaView(SchemaBaseView):
+    """
+    Displays schema for a specific database.
+    Supports HTML, JSON, and Markdown formats.
+    """
+
+    name = "database_schema"
+
+    async def get(self, request):
+        database_name = request.url_vars["database"]
+        format_ = request.url_vars.get("format") or "html"
+
+        # Check if database exists
+        if database_name not in self.ds.databases:
+            return self.format_error_response("Database not found", format_)
+
+        # Check view-database permission
+        await self.ds.ensure_permission(
+            action="view-database",
+            resource=DatabaseResource(database=database_name),
+            actor=request.actor,
+        )
+
+        schema = await self.get_database_schema(database_name)
+
+        if format_ == "json":
+            return self.format_json_response(
+                {"database": database_name, "schema": schema}
+            )
+        elif format_ == "md":
+            return self.format_markdown_response(f"Schema for {database_name}", schema)
+        else:
+            schemas = [{"database": database_name, "schema": schema}]
+            return await self.format_html_response(request, schemas)
+
+
+class TableSchemaView(SchemaBaseView):
+    """
+    Displays schema for a specific table.
+    Supports HTML, JSON, and Markdown formats.
+    """
+
+    name = "table_schema"
+
+    async def get(self, request):
+        database_name = request.url_vars["database"]
+        table_name = request.url_vars["table"]
+        format_ = request.url_vars.get("format") or "html"
+
+        # Check view-table permission
+        await self.ds.ensure_permission(
+            action="view-table",
+            resource=TableResource(database=database_name, table=table_name),
+            actor=request.actor,
+        )
+
+        # Get schema for the table
+        db = self.ds.databases[database_name]
+        result = await db.execute(
+            "select sql from sqlite_master where name = ? and sql is not null",
+            [table_name],
+        )
+        row = result.first()
+
+        # Return 404 if table doesn't exist
+        if not row or not row["sql"]:
+            return self.format_error_response("Table not found", format_)
+
+        schema = row["sql"]
+
+        if format_ == "json":
+            return self.format_json_response(
+                {"database": database_name, "table": table_name, "schema": schema}
+            )
+        elif format_ == "md":
+            return self.format_markdown_response(
+                f"Schema for {database_name}.{table_name}", schema
+            )
+        else:
+            schemas = [{"database": database_name, "schema": schema}]
+            return await self.format_html_response(
+                request, schemas, table_name=table_name
+            )

datasette/views/stored_queries.py

@@ -0,0 +1,483 @@
+from urllib.parse import parse_qsl, urlencode
+
+from datasette.resources import DatabaseResource, QueryResource
+from datasette.stored_queries import stored_query_to_dict
+from datasette.utils import sqlite3, tilde_decode
+from datasette.utils.asgi import Response
+
+from .base import BaseView, _error
+from .query_helpers import (
+    QueryValidationError,
+    _as_bool,
+    _as_optional_bool,
+    _block_framing,
+    _derived_query_parameters,
+    _json_or_form_payload,
+    _prepare_query_create,
+    _prepare_query_update,
+    _query_create_analysis_data,
+    _query_create_form_context,
+    _query_create_form_error_message,
+    _query_list_limit,
+)
+
+
+class QueryParametersView(BaseView):
+    name = "query-parameters"
+    has_json_alternate = False
+
+    async def get(self, request):
+        db = await self.ds.resolve_database(request)
+        if not await self.ds.allowed(
+            action="execute-sql",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        ):
+            return _block_framing(_error(["Permission denied: need execute-sql"], 403))
+
+        invalid_keys = set(request.args) - {"sql"}
+        if invalid_keys:
+            return _block_framing(
+                _error(
+                    ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))],
+                    400,
+                )
+            )
+        try:
+            parameters = _derived_query_parameters(request.args.get("sql") or "")
+        except QueryValidationError as ex:
+            return _block_framing(_error([ex.message], ex.status))
+        return _block_framing(Response.json({"ok": True, "parameters": parameters}))
+
+
+def _query_list_url(path, query_string, *, set_args=None, remove_args=None):
+    set_args = set_args or {}
+    remove_args = set(remove_args or ())
+    skip = set(set_args) | remove_args | {"_next"}
+    pairs = [
+        (key, value)
+        for key, value in parse_qsl(query_string, keep_blank_values=True)
+        if key not in skip
+    ]
+    for key, value in set_args.items():
+        if value not in (None, ""):
+            pairs.append((key, value))
+    return path + (("?" + urlencode(pairs)) if pairs else "")
+
+
+class QueryListView(BaseView):
+    name = "query-list"
+
+    async def database_name(self, request):
+        return (await self.ds.resolve_database(request)).name
+
+    def query_list_path(self, database):
+        return self.ds.urls.database(database) + "/-/queries"
+
+    async def get(self, request):
+        database = await self.database_name(request)
+        format_ = request.url_vars.get("format") or "html"
+        try:
+            limit = _query_list_limit(
+                request.args.get("_size"),
+                default=20 if format_ == "html" else 50,
+            )
+            is_write = _as_optional_bool(request.args.get("is_write"), "is_write")
+            is_private = _as_optional_bool(request.args.get("is_private"), "is_private")
+        except QueryValidationError as ex:
+            return _error([ex.message], ex.status)
+
+        page = await self.ds.list_queries(
+            database,
+            actor=request.actor,
+            limit=limit,
+            cursor=request.args.get("_next"),
+            q=request.args.get("q") or None,
+            is_write=is_write,
+            is_private=is_private,
+            source=request.args.get("source") or None,
+            owner_id=request.args.get("owner_id") or None,
+            include_private=True,
+        )
+        query_list_path = self.query_list_path(database)
+        next_url = None
+        if page.next:
+            pairs = [
+                (key, value)
+                for key, value in parse_qsl(
+                    request.query_string, keep_blank_values=True
+                )
+                if key != "_next"
+            ]
+            pairs.append(("_next", page.next))
+            next_url = "{}?{}".format(
+                query_list_path,
+                urlencode(pairs),
+            )
+
+        current_filters = {
+            "actor": request.actor,
+            "q": request.args.get("q") or None,
+            "is_write": is_write,
+            "is_private": is_private,
+            "source": request.args.get("source") or None,
+            "owner_id": request.args.get("owner_id") or None,
+        }
+
+        async def facet_count(field, value):
+            if current_filters[field] is not None and current_filters[field] != value:
+                return 0
+            filters = dict(current_filters)
+            filters[field] = value
+            return await self.ds.count_queries(database, **filters)
+
+        def facet_href(field, value):
+            if current_filters[field] == value:
+                return _query_list_url(
+                    query_list_path,
+                    request.query_string,
+                    remove_args=[field],
+                )
+            if current_filters[field] is not None:
+                return None
+            return _query_list_url(
+                query_list_path,
+                request.query_string,
+                set_args={field: str(int(value))},
+            )
+
+        async def facet_item(label, field, value):
+            count = await facet_count(field, value)
+            active = current_filters[field] == value
+            if not active and not count:
+                return None
+            return {
+                "label": label,
+                "count": count,
+                "href": facet_href(field, value) if active or count else None,
+                "active": active,
+            }
+
+        async def facet_items(items):
+            return [
+                item
+                for item in [
+                    await facet_item(label, field, value)
+                    for label, field, value in items
+                ]
+                if item is not None
+            ]
+
+        facets = [
+            {
+                "title": "Mode",
+                "items": await facet_items(
+                    [
+                        ("Read-only", "is_write", False),
+                        ("Writable", "is_write", True),
+                    ]
+                ),
+            },
+            {
+                "title": "Visibility",
+                "items": await facet_items(
+                    [
+                        ("Not private", "is_private", False),
+                        ("Private", "is_private", True),
+                    ]
+                ),
+            },
+        ]
+
+        data = {
+            "ok": True,
+            "database": database,
+            "database_color": (
+                self.ds.get_database(database).color if database is not None else None
+            ),
+            "queries": page.queries,
+            "next": page.next,
+            "next_url": next_url,
+            "has_more": page.has_more,
+            "limit": page.limit,
+            "show_private_note": any(query.is_private for query in page.queries),
+            "show_trusted_note": any(query.is_trusted for query in page.queries),
+            "query_list_path": query_list_path,
+            "show_database": database is None,
+            "facets": facets,
+            "filters": {
+                "q": request.args.get("q") or "",
+                "is_write": request.args.get("is_write") or "",
+                "is_private": request.args.get("is_private") or "",
+                "source": request.args.get("source") or "",
+                "owner_id": request.args.get("owner_id") or "",
+            },
+        }
+        if format_ == "json":
+            return Response.json(
+                {
+                    **data,
+                    "queries": [stored_query_to_dict(query) for query in page.queries],
+                }
+            )
+        return await self.render(
+            ["query_list.html"],
+            request,
+            data,
+        )
+
+
+class GlobalQueryListView(QueryListView):
+    name = "global-query-list"
+
+    async def database_name(self, request):
+        return None
+
+    def query_list_path(self, database):
+        return self.ds.urls.path("/-/queries")
+
+
+class QueryCreateView(BaseView):
+    name = "query-create"
+    has_json_alternate = False
+
+    async def _render_form(
+        self,
+        request,
+        db,
+        *,
+        sql="",
+        name="",
+        title="",
+        description="",
+        is_private=True,
+        status=200,
+    ):
+        response = await self.render(
+            ["query_create.html"],
+            request,
+            await _query_create_form_context(
+                self.ds,
+                request,
+                db,
+                sql=sql,
+                name=name,
+                title=title,
+                description=description,
+                is_private=is_private,
+            ),
+        )
+        response.status = status
+        return response
+
+    async def get(self, request):
+        db = await self.ds.resolve_database(request)
+        await self.ds.ensure_permission(
+            action="execute-sql",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        )
+        await self.ds.ensure_permission(
+            action="store-query",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        )
+
+        return await self._render_form(request, db, sql=request.args.get("sql") or "")
+
+
+class QueryCreateAnalyzeView(BaseView):
+    name = "query-create-analyze"
+    has_json_alternate = False
+
+    async def get(self, request):
+        db = await self.ds.resolve_database(request)
+        if not await self.ds.allowed(
+            action="execute-sql",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        ):
+            return _block_framing(_error(["Permission denied: need execute-sql"], 403))
+        if not await self.ds.allowed(
+            action="store-query",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        ):
+            return _block_framing(_error(["Permission denied: need store-query"], 403))
+
+        invalid_keys = set(request.args) - {"sql"}
+        if invalid_keys:
+            return _block_framing(
+                _error(
+                    ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))],
+                    400,
+                )
+            )
+        sql = request.args.get("sql") or ""
+        return _block_framing(
+            Response.json(
+                await _query_create_analysis_data(self.ds, db, sql, request.actor)
+            )
+        )
+
+
+class QueryStoreView(QueryCreateView):
+    name = "query-store"
+
+    async def _error_response(self, request, db, query_data, message, status):
+        message = _query_create_form_error_message(message)
+        self.ds.add_message(request, message, self.ds.ERROR)
+        return await self._render_form(
+            request,
+            db,
+            sql=query_data.get("sql") or "",
+            name=query_data.get("name") or "",
+            title=query_data.get("title") or "",
+            description=query_data.get("description") or "",
+            is_private=_as_bool(query_data.get("is_private", True)),
+            status=status,
+        )
+
+    async def post(self, request):
+        db = await self.ds.resolve_database(request)
+        if not await self.ds.allowed(
+            action="execute-sql",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        ):
+            return _error(["Permission denied: need execute-sql"], 403)
+        if not await self.ds.allowed(
+            action="store-query",
+            resource=DatabaseResource(db.name),
+            actor=request.actor,
+        ):
+            return _error(["Permission denied: need store-query"], 403)
+
+        is_json = False
+        query_data = {}
+        try:
+            data, is_json = await _json_or_form_payload(request)
+            if not isinstance(data, dict):
+                raise QueryValidationError("JSON must be a dictionary")
+            query_data = data.get("query") if is_json else data
+            if not isinstance(query_data, dict):
+                raise QueryValidationError("JSON must contain a query dictionary")
+            prepared = await _prepare_query_create(self.ds, request, db, query_data)
+        except QueryValidationError as ex:
+            if not is_json and isinstance(query_data, dict):
+                return await self._error_response(
+                    request, db, query_data, ex.message, ex.status
+                )
+            return _error([ex.message], ex.status)
+
+        prepared.pop("analysis")
+        name = prepared.pop("name")
+        try:
+            await self.ds.add_query(db.name, name, replace=False, **prepared)
+        except sqlite3.IntegrityError as ex:
+            if not is_json and isinstance(query_data, dict):
+                return await self._error_response(request, db, query_data, str(ex), 400)
+            return _error([str(ex)], 400)
+
+        query = await self.ds.get_query(db.name, name)
+        assert query is not None
+        if is_json:
+            return Response.json(
+                {"ok": True, "query": stored_query_to_dict(query)}, status=201
+            )
+        self.ds.add_message(request, "Query saved", self.ds.INFO)
+        return Response.redirect(self.ds.urls.path(self.ds.urls.table(db.name, name)))
+
+
+class QueryDefinitionView(BaseView):
+    name = "query-definition"
+
+    async def get(self, request):
+        db = await self.ds.resolve_database(request)
+        query_name = tilde_decode(request.url_vars["query"])
+        query = await self.ds.get_query(db.name, query_name)
+        if query is None:
+            return _error(["Query not found: {}".format(query_name)], 404)
+        if not await self.ds.allowed(
+            action="view-query",
+            resource=QueryResource(db.name, query_name),
+            actor=request.actor,
+        ):
+            return _error(["Permission denied"], 403)
+        return Response.json({"ok": True, "query": stored_query_to_dict(query)})
+
+
+class QueryUpdateView(BaseView):
+    name = "query-update"
+
+    async def post(self, request):
+        db = await self.ds.resolve_database(request)
+        query_name = tilde_decode(request.url_vars["query"])
+        existing = await self.ds.get_query(db.name, query_name)
+        if existing is None:
+            return _error(["Query not found: {}".format(query_name)], 404)
+        if not await self.ds.allowed(
+            action="update-query",
+            resource=QueryResource(db.name, query_name),
+            actor=request.actor,
+        ):
+            return _error(["Permission denied: need update-query"], 403)
+        if existing.is_trusted:
+            return _error(["Trusted queries cannot be updated using the API"], 403)
+
+        try:
+            data, _ = await _json_or_form_payload(request)
+            if not isinstance(data, dict):
+                raise QueryValidationError("JSON must be a dictionary")
+            invalid_keys = set(data) - {"update", "return"}
+            if invalid_keys:
+                raise QueryValidationError(
+                    "Invalid keys: {}".format(", ".join(invalid_keys))
+                )
+            update = data.get("update")
+            if not isinstance(update, dict):
+                raise QueryValidationError("JSON must contain an update dictionary")
+            if "sql" in update and not await self.ds.allowed(
+                action="execute-sql",
+                resource=DatabaseResource(db.name),
+                actor=request.actor,
+            ):
+                raise QueryValidationError(
+                    "Permission denied: need execute-sql", status=403
+                )
+            update_kwargs = await _prepare_query_update(
+                self.ds, request, db, existing, update
+            )
+        except QueryValidationError as ex:
+            return _error([ex.message], ex.status)
+
+        await self.ds.update_query(db.name, query_name, **update_kwargs)
+        if data.get("return"):
+            query = await self.ds.get_query(db.name, query_name)
+            assert query is not None
+            return Response.json(
+                {
+                    "ok": True,
+                    "query": stored_query_to_dict(query),
+                }
+            )
+        return Response.json({"ok": True})
+
+
+class QueryDeleteView(BaseView):
+    name = "query-delete"
+
+    async def post(self, request):
+        db = await self.ds.resolve_database(request)
+        query_name = tilde_decode(request.url_vars["query"])
+        existing = await self.ds.get_query(db.name, query_name)
+        if existing is None:
+            return _error(["Query not found: {}".format(query_name)], 404)
+        if not await self.ds.allowed(
+            action="delete-query",
+            resource=QueryResource(db.name, query_name),
+            actor=request.actor,
+        ):
+            return _error(["Permission denied: need delete-query"], 403)
+        await self.ds.remove_query(db.name, query_name)
+        return Response.json({"ok": True})

datasette/views/table.py

@@ -134,6 +134,22 @@ async def _redirect_if_needed(datasette, request, resolved):
         )
 
 
+async def _validate_column_types(datasette, database_name, table_name, rows):
+    """Validate row values against assigned column types. Returns list of error strings."""
+    ct_map = await datasette.get_column_types(database_name, table_name)
+    if not ct_map:
+        return []
+    errors = []
+    for row in rows:
+        for col_name, ct in ct_map.items():
+            if col_name not in row:
+                continue
+            error = await ct.validate(row[col_name], datasette)
+            if error:
+                errors.append(f"{col_name}: {error}")
+    return errors
+
+
 async def display_columns_and_rows(
     datasette,
     database_name,
@@ -163,6 +179,9 @@ async def display_columns_and_rows(
         )
     )
 
+    # Look up column types for this table
+    column_types_map = await datasette.get_column_types(database_name, table_name)
+
     column_details = {
         col.name: col for col in await db.table_column_details(table_name)
     }
@@ -179,16 +198,21 @@ async def display_columns_and_rows(
         else:
             type_ = column_details[r[0]].type
             notnull = column_details[r[0]].notnull
-        columns.append(
-            {
-                "name": r[0],
-                "sortable": r[0] in sortable_columns,
-                "is_pk": r[0] in pks_for_display,
-                "type": type_,
-                "notnull": notnull,
-                "description": column_descriptions.get(r[0]),
-            }
-        )
+        col_dict = {
+            "name": r[0],
+            "sortable": r[0] in sortable_columns,
+            "is_pk": r[0] in pks_for_display,
+            "type": type_,
+            "notnull": notnull,
+            "description": column_descriptions.get(r[0]),
+            "column_type": None,
+            "column_type_config": None,
+        }
+        ct = column_types_map.get(r[0])
+        if ct:
+            col_dict["column_type"] = ct.name
+            col_dict["column_type_config"] = ct.config
+        columns.append(col_dict)
 
     column_to_foreign_key_table = {
         fk["column"]: fk["other_table"]
@@ -227,22 +251,37 @@ async def display_columns_and_rows(
                 # already shown in the link column.
                 continue
 
-            # First let the plugins have a go
+            # First try column type render_cell, then plugins
             # pylint: disable=no-member
             plugin_display_value = None
-            for candidate in pm.hook.render_cell(
-                row=row,
-                value=value,
-                column=column,
-                table=table_name,
-                database=database_name,
-                datasette=datasette,
-                request=request,
-            ):
-                candidate = await await_me_maybe(candidate)
+            ct = column_types_map.get(column)
+            if ct:
+                candidate = await ct.render_cell(
+                    value=value,
+                    column=column,
+                    table=table_name,
+                    database=database_name,
+                    datasette=datasette,
+                    request=request,
+                )
                 if candidate is not None:
                     plugin_display_value = candidate
-                    break
+            if plugin_display_value is None:
+                for candidate in pm.hook.render_cell(
+                    row=row,
+                    value=value,
+                    column=column,
+                    table=table_name,
+                    pks=pks_for_display,
+                    database=database_name,
+                    datasette=datasette,
+                    request=request,
+                    column_type=ct,
+                ):
+                    candidate = await await_me_maybe(candidate)
+                    if candidate is not None:
+                        plugin_display_value = candidate
+                        break
             if plugin_display_value:
                 display_value = plugin_display_value
             elif isinstance(value, bytes):
@@ -330,6 +369,7 @@ async def display_columns_and_rows(
                 "is_pk": False,
                 "type": "",
                 "notnull": 0,
+                "is_special_link_column": True,
             }
         columns = [first_column] + columns
     return columns, cell_rows
@@ -421,6 +461,13 @@ class TableInsertView(BaseView):
                             i, '", "'.join(missing_pks)
                         )
                     )
+                null_pks = [pk for pk in pks_list if pk in row and row[pk] is None]
+                if null_pks:
+                    errors.append(
+                        'Row {} has null primary key column(s): "{}"'.format(
+                            i, '", "'.join(null_pks)
+                        )
+                    )
             invalid_columns = set(row.keys()) - columns
             if invalid_columns and not extras.get("alter"):
                 errors.append(
@@ -483,6 +530,13 @@ class TableInsertView(BaseView):
         if errors:
             return _error(errors, 400)
 
+        # Validate column types
+        ct_errors = await _validate_column_types(
+            self.ds, database_name, table_name, rows
+        )
+        if ct_errors:
+            return _error(ct_errors, 400)
+
         num_rows = len(rows)
 
         # No that we've passed pks to _validate_data it's safe to
@@ -550,7 +604,7 @@ class TableInsertView(BaseView):
                 method_all(rows, **kwargs)
 
         try:
-            rows = await db.execute_write_fn(insert_or_upsert_rows)
+            rows = await db.execute_write_fn(insert_or_upsert_rows, request=request)
         except Exception as e:
             return _error([str(e)])
         result = {"ok": True}
@@ -619,6 +673,122 @@ class TableUpsertView(TableInsertView):
         return await super().post(request, upsert=True)
 
 
+class TableSetColumnTypeView(BaseView):
+    name = "table-set-column-type"
+
+    def __init__(self, datasette):
+        self.ds = datasette
+
+    async def post(self, request):
+        try:
+            resolved = await self.ds.resolve_table(request)
+        except NotFound as e:
+            return _error([e.args[0]], 404)
+
+        database_name = resolved.db.name
+        table_name = resolved.table
+
+        if not await self.ds.allowed(
+            action="set-column-type",
+            resource=TableResource(database=database_name, table=table_name),
+            actor=request.actor,
+        ):
+            return _error(["Permission denied"], 403)
+
+        content_type = request.headers.get("content-type") or ""
+        if not content_type.startswith("application/json"):
+            return _error(["Invalid content-type, must be application/json"], 400)
+
+        try:
+            data = json.loads(await request.post_body())
+        except json.JSONDecodeError as e:
+            return _error(["Invalid JSON: {}".format(e)], 400)
+
+        if not isinstance(data, dict):
+            return _error(["JSON must be a dictionary"], 400)
+
+        invalid_keys = set(data.keys()) - {"column", "column_type"}
+        if invalid_keys:
+            return _error(
+                ['Invalid parameter: "{}"'.format('", "'.join(sorted(invalid_keys)))],
+                400,
+            )
+
+        if "column" not in data:
+            return _error(['"column" is required'], 400)
+        column = data["column"]
+        if not isinstance(column, str):
+            return _error(['"column" must be a string'], 400)
+
+        if "column_type" not in data:
+            return _error(['"column_type" is required'], 400)
+
+        column_details = await self.ds._get_resource_column_details(
+            database_name, table_name
+        )
+        if column not in column_details:
+            return _error(["Column not found: {}".format(column)], 400)
+
+        column_type_data = data["column_type"]
+        if column_type_data is None:
+            await self.ds.remove_column_type(database_name, table_name, column)
+            return Response.json(
+                {
+                    "ok": True,
+                    "database": database_name,
+                    "table": table_name,
+                    "column": column,
+                    "column_type": None,
+                },
+                status=200,
+            )
+
+        if not isinstance(column_type_data, dict):
+            return _error(['"column_type" must be an object or null'], 400)
+
+        invalid_column_type_keys = set(column_type_data.keys()) - {"type", "config"}
+        if invalid_column_type_keys:
+            return _error(
+                [
+                    'Invalid column_type parameter: "{}"'.format(
+                        '", "'.join(sorted(invalid_column_type_keys))
+                    )
+                ],
+                400,
+            )
+
+        if "type" not in column_type_data:
+            return _error(['"column_type.type" is required'], 400)
+        column_type = column_type_data["type"]
+        if not isinstance(column_type, str):
+            return _error(['"column_type.type" must be a string'], 400)
+
+        config = column_type_data.get("config")
+        if config is not None and not isinstance(config, dict):
+            return _error(['"column_type.config" must be a dictionary'], 400)
+
+        if column_type not in self.ds._column_types:
+            return _error(["Unknown column type: {}".format(column_type)], 400)
+
+        try:
+            await self.ds.set_column_type(
+                database_name, table_name, column, column_type, config
+            )
+        except ValueError as e:
+            return _error([str(e)], 400)
+
+        return Response.json(
+            {
+                "ok": True,
+                "database": database_name,
+                "table": table_name,
+                "column": column,
+                "column_type": {"type": column_type, "config": config},
+            },
+            status=200,
+        )
+
+
 class TableDropView(BaseView):
     name = "table-drop"
 
@@ -670,7 +840,7 @@ class TableDropView(BaseView):
         def drop_table(conn):
             sqlite_utils.Database(conn)[table_name].drop()
 
-        await db.execute_write_fn(drop_table)
+        await db.execute_write_fn(drop_table, request=request)
         await self.ds.track_event(
             DropTableEvent(
                 actor=request.actor, database=database_name, table=table_name
@@ -793,12 +963,12 @@ async def table_view_traced(datasette, request):
     try:
         resolved = await datasette.resolve_table(request)
     except TableNotFound as not_found:
-        # Was this actually a canned query?
-        canned_query = await datasette.get_canned_query(
-            not_found.database_name, not_found.table, request.actor
+        # Was this actually a stored query?
+        stored_query = await datasette.get_query(
+            not_found.database_name, not_found.table
         )
-        # If this is a canned query, not a table, then dispatch to QueryView instead
-        if canned_query:
+        # If this is a stored query, not a table, then dispatch to QueryView instead
+        if stored_query:
             return await QueryView()(request, datasette)
         else:
             raise
@@ -1420,6 +1590,10 @@ async def table_view_data(
         "Column names returned by this query"
         return columns
 
+    async def extra_all_columns():
+        "All columns in the table, regardless of _col/_nocol filtering"
+        return list(table_columns)
+
     async def extra_primary_keys():
         "Primary keys for this table"
         return pks
@@ -1492,6 +1666,50 @@ async def table_view_data(
     async def extra_display_rows(run_display_columns_and_rows):
         return run_display_columns_and_rows["rows"]
 
+    async def extra_render_cell():
+        "Rendered HTML for each cell using the render_cell plugin hook"
+        pks_for_display = pks if pks else (["rowid"] if not is_view else [])
+        col_names = [col[0] for col in results.description]
+        ct_map = await datasette.get_column_types(database_name, table_name)
+        rendered_rows = []
+        for row in rows:
+            rendered_row = {}
+            for value, column in zip(row, col_names):
+                ct = ct_map.get(column)
+                plugin_display_value = None
+                # Try column type render_cell first
+                if ct:
+                    candidate = await ct.render_cell(
+                        value=value,
+                        column=column,
+                        table=table_name,
+                        database=database_name,
+                        datasette=datasette,
+                        request=request,
+                    )
+                    if candidate is not None:
+                        plugin_display_value = candidate
+                if plugin_display_value is None:
+                    for candidate in pm.hook.render_cell(
+                        row=row,
+                        value=value,
+                        column=column,
+                        table=table_name,
+                        pks=pks_for_display,
+                        database=database_name,
+                        datasette=datasette,
+                        request=request,
+                        column_type=ct,
+                    ):
+                        candidate = await await_me_maybe(candidate)
+                        if candidate is not None:
+                            plugin_display_value = candidate
+                            break
+                if plugin_display_value:
+                    rendered_row[column] = str(plugin_display_value)
+            rendered_rows.append(rendered_row)
+        return rendered_rows
+
     async def extra_query():
         "Details of the underlying SQL query"
         return {
@@ -1499,6 +1717,58 @@ async def table_view_data(
             "params": params,
         }
 
+    async def extra_column_types():
+        "Column type assignments for this table"
+        ct_map = await datasette.get_column_types(database_name, table_name)
+        return {
+            col_name: {
+                "type": ct.name,
+                "config": ct.config,
+            }
+            for col_name, ct in ct_map.items()
+        }
+
+    async def extra_set_column_type_ui():
+        "Column type UI metadata for this table"
+        if is_view:
+            return None
+
+        if not await datasette.allowed(
+            action="set-column-type",
+            resource=TableResource(database=database_name, table=table_name),
+            actor=request.actor,
+        ):
+            return None
+
+        column_details = await datasette._get_resource_column_details(
+            database_name, table_name
+        )
+        ct_map = await datasette.get_column_types(database_name, table_name)
+        columns = {}
+        for column_name, column_detail in column_details.items():
+            current = ct_map.get(column_name)
+            columns[column_name] = {
+                "current": (
+                    {"type": current.name, "config": current.config}
+                    if current is not None
+                    else None
+                ),
+                "options": [
+                    {
+                        "name": name,
+                        "description": ct_cls.description,
+                    }
+                    for name, ct_cls in sorted(datasette._column_types.items())
+                    if datasette._column_type_is_applicable(ct_cls, column_detail)
+                ],
+            }
+        return {
+            "path": "{}/-/set-column-type".format(
+                datasette.urls.table(database_name, table_name)
+            ),
+            "columns": columns,
+        }
+
     async def extra_metadata():
         "Metadata about the table and database"
         tablemetadata = await datasette.get_resource_metadata(database_name, table_name)
@@ -1550,11 +1820,35 @@ async def table_view_data(
         ]
 
     async def extra_sorted_facet_results(extra_facet_results):
-        return sorted(
-            extra_facet_results["results"].values(),
-            key=lambda f: (len(f["results"]), f["name"]),
-            reverse=True,
-        )
+        facet_configs = table_metadata.get("facets", [])
+        if facet_configs:
+            # Build ordered list of facet names from metadata config
+            metadata_facet_names = []
+            for fc in facet_configs:
+                if isinstance(fc, str):
+                    metadata_facet_names.append(fc)
+                elif isinstance(fc, dict):
+                    metadata_facet_names.append(list(fc.values())[0])
+            metadata_order = {name: i for i, name in enumerate(metadata_facet_names)}
+            metadata_facets = []
+            request_facets = []
+            for f in extra_facet_results["results"].values():
+                if f["name"] in metadata_order:
+                    metadata_facets.append(f)
+                else:
+                    request_facets.append(f)
+            metadata_facets.sort(key=lambda f: metadata_order[f["name"]])
+            request_facets.sort(
+                key=lambda f: (len(f["results"]), f["name"]),
+                reverse=True,
+            )
+            return metadata_facets + request_facets
+        else:
+            return sorted(
+                extra_facet_results["results"].values(),
+                key=lambda f: (len(f["results"]), f["name"]),
+                reverse=True,
+            )
 
     async def extra_table_definition():
         return await db.get_table_definition(table_name)
@@ -1654,8 +1948,10 @@ async def table_view_data(
             "is_view",
             "private",
             "primary_keys",
+            "all_columns",
             "expandable_columns",
             "form_hidden_args",
+            "set_column_type_ui",
         ]
     }
 
@@ -1674,13 +1970,17 @@ async def table_view_data(
         extra_human_description_en,
         extra_next_url,
         extra_columns,
+        extra_all_columns,
         extra_primary_keys,
         run_display_columns_and_rows,
         extra_display_columns,
         extra_display_rows,
+        extra_render_cell,
         extra_debug,
         extra_request,
         extra_query,
+        extra_column_types,
+        extra_set_column_type_ui,
         extra_metadata,
         extra_extras,
         extra_database,
@@ -1714,7 +2014,18 @@ async def table_view_data(
         }
     )
     raw_sqlite_rows = rows[:page_size]
-    data["rows"] = [dict(r) for r in raw_sqlite_rows]
+    # Apply transform_value for columns with assigned types
+    ct_map = await datasette.get_column_types(database_name, table_name)
+    transformed_rows = []
+    for r in raw_sqlite_rows:
+        row_dict = dict(r)
+        for col_name, ct in ct_map.items():
+            if col_name in row_dict:
+                row_dict[col_name] = await ct.transform_value(
+                    row_dict[col_name], datasette
+                )
+        transformed_rows.append(row_dict)
+    data["rows"] = transformed_rows
 
     if context_for_html_hack:
         data.update(extra_context_from_filters)

docs/authentication.rst

@@ -33,7 +33,7 @@ The one exception is the "root" account, which you can sign into while using Dat
 The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including:
 
 * All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.)
-* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``drop-table``)
+* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``drop-table``)
 * Debug permissions (``permissions-debug``, ``debug-menu``)
 * Any custom permissions defined by plugins
 
@@ -83,12 +83,45 @@ Datasette's built-in view actions (``view-database``, ``view-table`` etc) are al
 
 Other actions, including those introduced by plugins, will default to *deny*.
 
+.. _authentication_default_deny:
+
+Denying all permissions by default
+----------------------------------
+
+By default, Datasette allows unauthenticated access to view databases, tables, and execute SQL queries.
+
+You may want to run Datasette in a mode where **all** access is denied by default, and you explicitly grant permissions only to authenticated users, either using the :ref:`--root mechanism <authentication_root>` or through :ref:`configuration file rules <authentication_permissions_config>` or plugins.
+
+Use the ``--default-deny`` command-line option to run Datasette in this mode::
+
+    datasette --default-deny data.db --root
+
+With ``--default-deny`` enabled:
+
+* Anonymous users are denied access to view the instance, databases, tables, and queries
+* Authenticated users are also denied access unless they're explicitly granted permissions
+* The root user (when using ``--root``) still has access to everything
+* You can grant permissions using :ref:`configuration file rules <authentication_permissions_config>` or plugins
+
+For example, to allow only a specific user to access your instance::
+
+    datasette --default-deny data.db --config datasette.yaml
+
+Where ``datasette.yaml`` contains:
+
+.. code-block:: yaml
+
+    allow:
+      id: alice
+
+This configuration will deny access to everyone except the user with ``id`` of ``alice``.
+
 .. _authentication_permissions_explained:
 
 How permissions are resolved
 ----------------------------
 
-Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``. 
+Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``.
 
 ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified.
 
@@ -435,7 +468,7 @@ You can control the following:
 * Access to the entire Datasette instance
 * Access to specific databases
 * Access to specific tables and views
-* Access to specific :ref:`canned_queries`
+* Access to specific :ref:`queries <queries>`
 
 If a user has permission to view a table they will be able to view that table, independent of if they have permission to view the database or instance that the table exists within.
 
@@ -608,12 +641,12 @@ This works for SQL views as well - you can list their names in the ``"tables"``
 
 .. _authentication_permissions_query:
 
-Access to specific canned queries
----------------------------------
+Access to specific queries
+--------------------------
 
-:ref:`canned_queries` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important.
+:ref:`Queries <queries>` allow you to configure named SQL queries in your ``datasette.yaml`` that can be executed by users. These queries can be set up to both read and write to the database, so controlling who can execute them can be important.
 
-To limit access to the ``add_name`` canned query in your ``dogs.db`` database to just the :ref:`root user<authentication_root>`:
+To limit access to the ``add_name`` query in your ``dogs.db`` database to just the :ref:`root user<authentication_root>`:
 
 .. [[[cog
     config_example(cog, """
@@ -853,6 +886,8 @@ To grant ``create-table`` to the user with ``id`` of ``editor`` for the ``docs``
         }
 .. [[[end]]]
 
+Other table-scoped write permissions, including ``set-column-type``, can be configured in the same place.
+
 And for ``insert-row`` against the ``reports`` table in that ``docs`` database:
 
 .. [[[cog
@@ -985,7 +1020,7 @@ You can also restrict permissions such that they can only be used within specifi
 
 The resulting token will only be able to insert rows, and only to tables in the ``mydatabase`` database.
 
-Finally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries <canned_queries>` - within a specific database::
+Finally, you can restrict permissions to individual resources - tables, SQL views and :ref:`named queries <queries>` - within a specific database::
 
     datasette create-token root --resource mydatabase mytable insert-row
 
@@ -1039,6 +1074,7 @@ cannot grant new access. If the underlying actor is denied by ``allow`` rules in
 ``datasette.yaml`` or by a plugin, a token that lists that resource in its
 ``"_r"`` section will still be denied.
 
+To create tokens with restrictions in Python code, use the :ref:`TokenRestrictions <TokenRestrictions>` builder and pass it to :ref:`datasette.create_token() <datasette_create_token>`.
 
 .. _permissions_plugins:
 
@@ -1249,12 +1285,46 @@ Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.i
 view-query
 ----------
 
-Actor is allowed to view (and execute) a :ref:`canned query <canned_queries>` page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size - this includes executing :ref:`canned_queries_writable`.
+Actor is allowed to view a stored query page, e.g. https://latest.datasette.io/fixtures/pragma_cache_size. Executing an untrusted stored query also requires ``execute-sql`` or the relevant write permissions; :ref:`trusted stored queries <trusted_stored_queries>` can execute with ``view-query`` alone.
 
 ``resource`` - ``datasette.resources.QueryResource(database, query)``
     ``database`` is the name of the database (string)
-    
-    ``query`` is the name of the canned query (string)
+
+    ``query`` is the name of the query (string)
+
+.. _actions_store_query:
+
+store-query
+-----------
+
+Actor is allowed to create stored queries against a database.
+
+``resource`` - ``datasette.resources.DatabaseResource(database)``
+    ``database`` is the name of the database (string)
+
+.. _actions_update_query:
+
+update-query
+------------
+
+Actor is allowed to update a stored query.
+
+``resource`` - ``datasette.resources.QueryResource(database, query)``
+    ``database`` is the name of the database (string)
+
+    ``query`` is the name of the query (string)
+
+.. _actions_delete_query:
+
+delete-query
+------------
+
+Actor is allowed to delete a stored query.
+
+``resource`` - ``datasette.resources.QueryResource(database, query)``
+    ``database`` is the name of the database (string)
+
+    ``query`` is the name of the query (string)
 
 .. _actions_insert_row:
 
@@ -1309,6 +1379,18 @@ alter-table
 
 Actor is allowed to alter a database table.
 
+``resource`` - ``datasette.resources.TableResource(database, table)``
+    ``database`` is the name of the database (string)
+
+    ``table`` is the name of the table (string)
+
+.. _actions_set_column_type:
+
+set-column-type
+---------------
+
+Actor is allowed to set assigned :ref:`column types <table_configuration_column_types>` for columns in a table.
+
 ``resource`` - ``datasette.resources.TableResource(database, table)``
     ``database`` is the name of the database (string)
 
@@ -1331,13 +1413,23 @@ Actor is allowed to drop a database table.
 execute-sql
 -----------
 
-Actor is allowed to run arbitrary SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100
+Actor is allowed to run arbitrary read-only SQL queries against a specific database, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100
 
 ``resource`` - ``datasette.resources.DatabaseResource(database)``
     ``database`` is the name of the database (string)
 
 See also :ref:`the default_allow_sql setting <setting_default_allow_sql>`.
 
+.. _actions_execute_write_sql:
+
+execute-write-sql
+-----------------
+
+Actor is allowed to run arbitrary writable SQL queries against a specific database, subject to table-level write permissions such as ``insert-row``, ``update-row`` and ``delete-row``.
+
+``resource`` - ``datasette.resources.DatabaseResource(database)``
+    ``database`` is the name of the database (string)
+
 .. _actions_permissions_debug:
 
 permissions-debug
@@ -1350,4 +1442,4 @@ Actor is allowed to view the ``/-/permissions`` debug tools.
 debug-menu
 ----------
 
-Controls if the various debug pages are displayed in the navigation menu.
+Controls if the various debug pages are displayed in the jump menu.

docs/changelog.rst

@@ -4,29 +4,294 @@
 Changelog
 =========
 
+.. _v1_0_unreleased:
+
+Unreleased
+----------
+
+Stored queries
+~~~~~~~~~~~~~~
+
+- The previous "canned queries" feature has been renamed and expanded into :ref:`stored queries <stored_queries>`. Queries configured in ``datasette.yaml`` are now loaded into a new ``queries`` table in Datasette's :ref:`internal database <internals_internal_schema>`, alongside user-created stored queries. (:issue:`2735`)
+- New stored query management APIs: ``datasette.add_query()``, ``datasette.update_query()``, ``datasette.remove_query()``, ``datasette.get_query()``, ``datasette.list_queries()`` and ``datasette.count_queries()``. These replace the removed ``datasette.get_canned_query()`` and ``datasette.get_canned_queries()`` methods. (:issue:`2735`)
+- Users with :ref:`store-query <actions_store_query>` and :ref:`execute-sql <actions_execute_sql>` permission can create stored queries from the SQL query page or the new ``GET /<database>/-/queries/store`` form. (:issue:`2735`)
+- The database page now shows a count and preview of stored queries, capped at five, and links to new paginated query browsers at ``/-/queries`` and ``/<database>/-/queries``. Those browsers support search. (:issue:`2735`)
+- Stored queries created by users default to private and untrusted. Private stored queries can only be viewed, updated or deleted by their owner, even if another actor has broad ``view-query``, ``update-query`` or ``delete-query`` permission. Untrusted stored queries execute using the permissions of the actor running them. See :ref:`stored_queries` and :ref:`trusted_stored_queries` for details. (:issue:`2735`)
+- New ``store-query``, ``update-query`` and ``delete-query`` permissions, plus updated semantics for :ref:`view-query <actions_view_query>`. Trusted stored queries can still execute with ``view-query`` alone; untrusted read queries also require :ref:`execute-sql <actions_execute_sql>` and untrusted writable queries require :ref:`execute-write-sql <actions_execute_write_sql>` plus the relevant table-level write permissions. (:issue:`2735`)
+
+Write SQL UI
+~~~~~~~~~~~~
+
+- New "Write to this database" interface at ``/<database>/-/execute-write`` for running arbitrary writable SQL against mutable databases. The form extracts named parameters, analyzes the SQL, shows the table operations that will be attempted and links to a newly inserted row when a single-row insert succeeds. (:issue:`2742`)
+- Added the new :ref:`execute-write-sql <actions_execute_write_sql>` permission for running arbitrary writable SQL. Execution is also gated by table-level permissions such as :ref:`insert-row <actions_insert_row>`, :ref:`update-row <actions_update_row>` and :ref:`delete-row <actions_delete_row>`, and writes to attached databases are rejected. (:issue:`2742`)
+
+Plugin API changes
+~~~~~~~~~~~~~~~~~~
+
+- The ``top_canned_query()`` plugin hook has been renamed to :ref:`top_stored_query() <plugin_hook_top_stored_query>`. (:issue:`2747`)
+- The ``canned_queries()`` plugin hook has been removed. Plugins can use the new :ref:`stored query management methods <datasette_stored_queries>` together with :ref:`startup() <plugin_hook_startup>` to register queries. (:issue:`2735`)
+
+Bug fixes
+~~~~~~~~~
+
+- Fixed a bug where visiting ``/<database>/-/query`` without a ``?sql=`` parameter returned a 500 error. (:issue:`2743`)
+
+.. _v1_0_a30:
+
+1.0a30 (2026-05-24)
+-------------------
+
+The "Jump to" menu, activated by hitting ``/`` or through the application menu, can now be extended by plugins.
+
+- New "Jump to..." menu item, always visible, for triggering the previously undocumented ``/`` menu. (:issue:`2725`)
+- The ``/`` jump-to search interface now covers databases, views, canned queries and plugin-provided items in addition to tables. The endpoint backing it has been renamed from ``/-/tables`` to ``/-/jump``.
+- New :ref:`plugin_hook_jump_items_sql` plugin hook, allowing plugins to contribute additional items to the jump-to menu by returning SQL. ``JumpSQL`` queries run against Datasette's internal database by default, or can target another database using the optional ``database=`` argument. (:issue:`2731`)
+- ``datasette.jump.JumpSQL.menu_item()`` is a shortcut for adding individual jump menu items that are not backed by resources in the internal catalog.
+- New :ref:`javascript_plugins_makeJumpSections` JavaScript plugin hook, allowing plugins to add custom blank-state sections to the jump-to menu before the user has typed a query.
+- Debug menu links now appear in the jump-to menu instead of the top-right app menu, with descriptions for each debug item.
+- Dropped Janus as a dependency, previously used to manage the write queue. This should not have any impact on plugin developers or end-users. (:issue:`1752`)
+- Fixed a bug where stale tables and other related resources were not removed from ``catalog_*`` tables when a database was removed. (:issue:`2723`)
+- New documented :ref:`datasette.fixtures.populate_fixture_database(conn) <datasette_fixtures_populate_fixture_database>` helper for creating the fixture database tables used by Datasette's own tests, intended for plugin test suites.
+- Keyboard accessibility and ARIA roles for actions menus, thanks `pintaste <https://github.com/pintaste>`__. (:pr:`2727`)
+
+.. _v1_0_a29:
+
+1.0a29 (2026-05-12)
+-------------------
+
+- New ``TokenRestrictions.abbreviated(datasette)`` :ref:`utility method <TokenRestrictions>` for creating ``"_r"`` dictionaries. (:issue:`2695`)
+- Table headers and column options are now visible even if a table contains zero rows. (:issue:`2701`)
+- Fixed bug with display of column actions dialog on Mobile Safari. (:issue:`2708`)
+- Fixed bug where tests could crash with a segfault due to a race condition between ``Datasette.close()`` and ``Datasette.close()``. (:issue:`2709`)
+
+.. _v1_0_a28:
+
+1.0a28 (2026-04-16)
+-------------------
+
+- Fixed a compatibility bug introduced in 1.0a27 where ``execute_write_fn()`` callbacks with a parameter name other than ``conn`` were seeing errors. (:issue:`2691`)
+- The :ref:`database.close() <database_close>` method now also shuts down the write connection for that database.
+- New :ref:`datasette.close() <datasette_close>` method for closing down all databases and resources associated with a Datasette instance. This is called automatically when the server shuts down. (:pr:`2693`)
+- Datasette now includes a pytest plugin which automatically calls ``datasette.close()`` on temporary instances created in function-scoped fixtures and during tests. See :ref:`testing_plugins_autoclose` for details. This helps avoid running out of file descriptors in plugin test suites that were written before the ``Database(is_temp_disk=True)`` feature introduced in Datasette 1.0a27. (:issue:`2692`)
+
+.. _v1_0_a27:
+
+1.0a27 (2026-04-15)
+-------------------
+
+CSRF protection no longer uses CSRF tokens
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Datasette's token-based CSRF protection has been replaced with a mechanism based on the ``Sec-Fetch-Site`` and ``Origin`` request headers, which are `supported by all modern browsers <https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Sec-Fetch-Site>`__. See `this article by Filippo Valsorda <https://words.filippo.io/csrf/>`__ for more details of this approach. This removes the need for CSRF tokens in forms and AJAX requests. (:pr:`2689`)
+
+``RenameTableEvent`` when a table is renamed
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Renaming a table within Datasette will now fire a new :class:`~datasette.events.RenameTableEvent`, which plugins can use to react by updating ACL records or re-assigning comments or other associated records to the new table name. (:issue:`2681`)
+
+This event will not be fired if the table is renamed by SQL running in some other process.
+
+The ``datasette.track_event()`` method can now be called from within a write operation (using :ref:`database.execute_write() <database_execute_write>` and related methods) and the event will be fired after the write transaction has successfully committed. (:pr:`2682`)
+
+Other changes
+~~~~~~~~~~~~~
+
+- New :ref:`actor= parameter <internals_datasette_client_actor>` for ``datasette.client`` methods, allowing internal requests to be made as a specific actor. This is particularly useful for writing automated tests. (:pr:`2688`)
+- New ``Database(is_temp_disk=True)`` option, used internally for the internal database. This helps resolve intermittent database locked errors caused by the internal database being in-memory as opposed to on-disk. (:issue:`2683`) (:pr:`2684`)
+- The ``/<database>/<table>/-/upsert`` API (:ref:`docs <TableUpsertView>`) now rejects rows with ``null`` primary key values. (:issue:`1936`)
+- Improved example in the API explorer for the ``/-/upsert`` endpoint (:ref:`docs <TableUpsertView>`). (:issue:`1936`)
+- The ``/<database>.json`` endpoint now includes an ``"ok": true`` key, for consistency with other JSON API responses.
+- :ref:`call_with_supported_arguments() <internals_utils_call_with_supported_arguments>` is now documented as a supported public API. (:pr:`2678`)
+
+.. _v1_0_a26:
+
+1.0a26 (2026-03-18)
+-------------------
+
+New ``column_types`` system
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Table columns can now have custom column types assigned to them, using the new ``column_types`` table configuration option or at runtime using a new UI and ``POST /<database>/<table>/-/set-column-type`` JSON API.
+
+Built-in column types include ``url``, ``email``, and ``json``, and plugins can register additional types using the new :ref:`register_column_types() <plugin_register_column_types>` plugin hook. (:issue:`2664`, :issue:`2671`)
+
+Column types can customize HTML rendering, validate values written through the insert, update, and upsert APIs, and transform values returned by the JSON API. They can optionally restrict themselves to specific SQLite column types using ``sqlite_types``. This feature also introduces a new :ref:`set-column-type <actions_set_column_type>` permission for assigning column types to a table. (:issue:`2672`)
+
+The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook now receives a ``column_type`` argument containing the assigned type instance, and a column type's own ``render_cell()`` method takes priority over the plugin hook chain.
+
+The `datasette-files <https://github.com/datasette/datasette-files>`__ plugin will be the first to use this new feature.
+
+UI for selecting columns and their order
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Table and view pages now include a dialog for selecting and re-ordering visible columns. (:issue:`2661`)
+
+Other changes
+~~~~~~~~~~~~~
+
+- Fixed ``allowed_resources("view-query", actor)`` so actor-specific canned queries are returned correctly. Any plugin that defines a ``resources_sql()`` method on a ``Resource`` subclass needs to update to the new signature, see :ref:`the resources_sql() method<plugin_resources_sql>` documentation for details.
+- Column actions can now be accessed in mobile view via a new "Column actions" button. Previously they were not available on mobile because table headers are not displayed there. (:issue:`2669`, :issue:`2670`)
+- Row pages now render foreign key values as links to the referenced row. (:issue:`1592`)
+- The ``startup()`` plugin hook now fires after metadata and internal schema tables have been populated, so plugins can reliably inspect that state during startup. (:issue:`2666`)
+
+.. _v1_0_a25:
+
+1.0a25 (2026-02-25)
+-------------------
+
+``write_wrapper()`` plugin hook for intercepting write operations
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A new :ref:`write_wrapper() <plugin_hook_write_wrapper>` plugin hook allows plugins to intercept and wrap database write operations. (:pr:`2636`)
+
+Plugins implement the hook as a generator-based context manager:
+
+.. code-block:: python
+
+    @hookimpl
+    def write_wrapper(datasette, database, request):
+        def wrapper(conn):
+            # Setup code runs before the write
+            yield
+            # Cleanup code runs after the write
+
+        return wrapper
+
+``register_token_handler()`` plugin hook for custom API token backends
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A new :ref:`register_token_handler() <plugin_hook_register_token_handler>` plugin hook allows plugins to provide custom token backends for API authentication. (:pr:`2650`)
+
+This includes a **backwards incompatible change**: the ``datasette.create_token()`` internal  method is now an ``async`` method. Consult the :ref:`upgrade guide <upgrade_guide_v1_a25>` for details on how to update your code.
+
+``render_cell()`` now receives a ``pks`` parameter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook now receives a ``pks`` parameter containing the list of primary key column names for the table being rendered. This avoids plugins needing to make redundant async calls to look up primary keys. (:pr:`2641`)
+
+Other changes
+~~~~~~~~~~~~~
+
+- Facets defined in metadata now preserve their configured order, instead of being sorted by result count. Request-based facets added via the ``_facet`` parameter are still sorted by result count and appear after metadata-defined facets. (:issue:`2647`)
+- Fixed ``--reload`` incorrectly interpreting the ``serve`` command as a file argument. Thanks, `Daniel Bates <https://github.com/danielalanbates>`__. (:pr:`2646`)
+
+.. _v1_0_a24:
+
+1.0a24 (2026-01-29)
+-------------------
+
+``request.form()`` method for POST data and file uploads
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Datasette now includes a ``request.form()`` method for parsing form submissions, including handling file uploads. (:pr:`2626`)
+
+This supports both ``application/x-www-form-urlencoded`` and ``multipart/form-data`` content types, and uses a new streaming multipart parser that processes uploads without buffering entire request bodies in memory.
+
+.. code-block:: python
+
+    # Parse form fields (files are discarded by default)
+    form = await request.form()
+    username = form["username"]
+
+    # Parse form fields AND file uploads
+    form = await request.form(files=True)
+    uploaded = form["avatar"]
+    content = await uploaded.read()
+
+The returned :ref:`FormData <internals_formdata>` object provides dictionary-style access with support for multiple values per key via ``form.getlist("key")``. Uploaded files are represented as :ref:`UploadedFile <internals_uploadedfile>` objects with ``filename``, ``content_type``, ``size`` properties and async ``read()`` and ``seek()`` methods.
+
+Files smaller than 1MB are held in memory; larger files automatically spill to temporary files on disk. Configurable limits control maximum file size, request size, field counts and more.
+
+Several internal views (permissions debug, messages debug, create token) now use ``request.form()`` instead of ``request.post_vars()``.
+
+``request.post_vars()`` remains available for backwards compatibility but is no longer the recommended API for handling POST data.
+
+``render_cell`` and ``foreign_key_tables`` extras for the JSON API
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The table JSON API now supports ``?_extra=render_cell``, which returns the rendered HTML for each cell as produced by the :ref:`render_cell plugin hook <plugin_hook_render_cell>`. Only columns whose rendered output differs from the default are included. (:issue:`2619`)
+
+The row JSON API also gains ``?_extra=render_cell`` and ``?_extra=foreign_key_tables`` extras, bringing it closer to parity with the table API.
+
+The row JSON API now returns ``"ok": true`` in its response, for consistency with the table API.
+
+``uv run pytest`` with a ``dev=`` dependency group
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The recommended development environment for Datasette now uses `uv <https://github.com/astral-sh/uv>`__. You can now set up a development environment and run the test suite with just ``uv run pytest`` — no manual virtualenv or ``pip install`` step required. (:issue:`2611`)
+
+Other changes
+~~~~~~~~~~~~~
+
+- Plugins that raise ``datasette.utils.StartupError()`` during startup now display a clean error message instead of a full traceback. (:issue:`2624`)
+- Schema refreshes are now throttled to at most once per second, providing a small performance increase. (:issue:`2629`)
+- Minor performance improvement to ``remove_infinites`` — rows without infinity values now skip the list/dict reconstruction step. (:issue:`2629`)
+- Filter inputs and the search input no longer trigger unwanted zoom on iOS Safari. Thanks, `Daniel Olasubomi Sobowale <https://github.com/bowale-os>`__. (:issue:`2346`)
+- ``table_names()`` and ``get_all_foreign_keys()`` now return results in deterministic sorted order. (:issue:`2628`)
+- Switched linting to `ruff <https://github.com/astral-sh/ruff>`__ and fixed all lint errors. (:issue:`2630`)
+
+.. _v1_0_a23:
+
+1.0a23 (2025-12-02)
+-------------------
+
+- Fix for bug where a stale database entry in ``internal.db`` could cause a 500 error on the homepage. (:issue:`2605`)
+- Cosmetic improvement to ``/-/actions`` page. (:issue:`2599`)
+
+.. _v1_0_a22:
+
+1.0a22 (2025-11-13)
+-------------------
+
+- ``datasette serve --default-deny`` option for running Datasette configured to  :ref:`deny all permissions by default <authentication_default_deny>`. (:issue:`2592`)
+- ``datasette.is_client()`` method for detecting if code is :ref:`executing inside a datasette.client request <internals_datasette_is_client>`. (:issue:`2594`)
+- ``datasette.pm`` property can now be used to :ref:`register and unregister plugins in tests <testing_plugins_register_in_test>`. (:issue:`2595`)
+
+.. _v1_0_a21:
+
+1.0a21 (2025-11-05)
+-------------------
+
+- Fixes an **open redirect** security issue: Datasette instances would redirect to ``example.com/foo/bar`` if you accessed the path ``//example.com/foo/bar``. Thanks to `James Jefferies <https://github.com/jamesjefferies>`__ for the fix. (:issue:`2429`)
+- Fixed ``datasette publish cloudrun`` to work with changes to the underlying Cloud Run architecture. (:issue:`2511`)
+- New ``datasette --get /path --headers`` option for inspecting the headers returned by a path. (:issue:`2578`)
+- New ``datasette.client.get(..., skip_permission_checks=True)`` parameter to bypass permission checks when making requests using the internal client. (:issue:`2583`)
+
+.. _v0_65_2:
+
+0.65.2 (2025-11-05)
+-------------------
+
+- Fixes an **open redirect** security issue: Datasette instances would redirect to ``example.com/foo/bar`` if you accessed the path ``//example.com/foo/bar``. Thanks to `James Jefferies <https://github.com/jamesjefferies>`__ for the fix. (:issue:`2429`)
+- Upgraded for compatibility with Python 3.14.
+- Fixed ``datasette publish cloudrun`` to work with changes to the underlying Cloud Run architecture. (:issue:`2511`)
+- Minor upgrades to fix warnings, including ``pkg_resources`` deprecation.
+
 .. _v1_0_a20:
 
-UNRELEASED 1.0a20 (2025-??-??)
-------------------------------
+1.0a20 (2025-11-03)
+-------------------
 
-This alpha introduces a major breaking change prior to the 1.0 release of Datasette concerning Datasette's permission system.
+This alpha introduces a major breaking change prior to the 1.0 release of Datasette concerning how Datasette's permission system works.
 
 Permission system redesign
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Previously the permission system worked using ``datasette.permission_allowed()`` checks which consulted all available plugins in turn to determine whether a given actor was allowed to perform a given action on a given resource.
 
-This approach could become prohibitively expensive for large lists of items - for example to determine the list of tables that a user could view in a large Datasette instance, where the plugin hooks would be called N times for N tables.
+This approach could become prohibitively expensive for large lists of items - for example to determine the list of tables that a user could view in a large Datasette instance each plugin implementation of that hook would be fired for every table.
 
-The new system instead uses SQL queries against Datasette's internal :ref:`catalog tables <internals_internal>` to derive the list of resources for which an actor has permission for a given action.
+The new design uses SQL queries against Datasette's internal :ref:`catalog tables <internals_internal>` to derive the list of resources for which an actor has permission for a given action. This turns an N x M problem (N resources, M plugins) into a single SQL query.
 
-Plugins can use the new :ref:`plugin_hook_permission_resources_sql` hook to return SQL fragments which will influence the construction of that query.
+Plugins can use the new :ref:`plugin_hook_permission_resources_sql` hook to return SQL fragments which will be used as part of that query.
 
-Affected plugins should make the following changes:
+Plugins that use any of the following features will need to be updated to work with this and following alphas (and Datasette 1.0 stable itself):
 
-- Replace calls to ``datasette.permission_allowed()`` with calls to the new :ref:`datasette.allowed() <datasette_allowed>` method. The new method takes a ``resource=`` parameter which should be an instance of a ``Resource`` subclass, as described in the method documentation.
-- The ``permission_allowed()`` plugin hook has been removed in favor of the new :ref:`permission_resources_sql() <plugin_hook_permission_resources_sql>` hook.
-- The ``register_permissions()`` plugin hook has been removed in favor of :ref:`register_actions() <plugin_register_actions>`.
+- Checking permissions with ``datasette.permission_allowed()`` - this method has been replaced with :ref:`datasette.allowed() <datasette_allowed>`.
+- Implementing the ``permission_allowed()`` plugin hook - this hook has been removed in favor of :ref:`permission_resources_sql() <plugin_hook_permission_resources_sql>`.
+- Using ``register_permissions()`` to register permissions - this hook has been removed in favor of :ref:`register_actions() <plugin_register_actions>`.
+
+Consult the :ref:`v1.0a20 upgrade guide <upgrade_guide_v1_a20>` for further details on how to upgrade affected plugins.
 
 Plugins can now make use of two new internal methods to help resolve permission checks:
 
@@ -40,22 +305,16 @@ Related changes:
 - Permission debugging improvements:
 
   - The ``/-/allowed`` endpoint shows resources the user is allowed to interact with for different actions.
-
   - ``/-/rules`` shows the raw allow/deny rules that apply to different permission checks.
-
   - ``/-/actions`` lists every available action.
-
   - ``/-/check`` can be used to try out different permission checks for the current actor.
 
 Other changes
 ~~~~~~~~~~~~~
 
 - The internal ``catalog_views`` table now tracks SQLite views alongside tables in the introspection database. (:issue:`2495`)
-
 - Hitting the ``/`` brings up a search interface for navigating to tables that the current user can view. A new ``/-/tables`` endpoint supports this functionality. (:issue:`2523`)
-
 - Datasette attempts to detect some configuration errors on startup.
-
 - Datasette now supports Python 3.14 and no longer tests against Python 3.9.
 
 .. _v1_0_a19:
@@ -75,7 +334,7 @@ Other changes
 - Fixed bug where ``link:`` HTTP headers used invalid syntax. (:issue:`2470`)
 - No longer tested against Python 3.8. Now tests against Python 3.13.
 - FTS tables are now hidden by default if they correspond to a content table. (:issue:`2477`)
-- Fixed bug with foreign key links to rows in databases with filenames containing a special character. Thanks, `Jack Stratton <https://github.com/phroa>`__. (`#2476 <https://github.com/simonw/datasette/pull/2476>`__)
+- Fixed bug with foreign key links to rows in databases with filenames containing a special character. Thanks, `Jack Stratton <https://github.com/phroa>`__. (:pr:`2476`)
 
 .. _v1_0_a17:
 
@@ -118,7 +377,7 @@ Other changes
 This release focuses on performance, in particular against large tables, and introduces some minor breaking changes for CSS styling in Datasette plugins.
 
 - Removed the unit conversions feature and its dependency, Pint. This means Datasette is now compatible with the upcoming Python 3.13. (:issue:`2400`, :issue:`2320`)
-- The ``datasette --pdb`` option now uses the `ipdb <https://github.com/gotcha/ipdb>`__ debugger if it is installed. You can install it using ``datasette install ipdb``. Thanks, `Tiago Ilieve <https://github.com/myhro>`__. (`#2342 <https://github.com/simonw/datasette/pull/2342>`__)
+- The ``datasette --pdb`` option now uses the `ipdb <https://github.com/gotcha/ipdb>`__ debugger if it is installed. You can install it using ``datasette install ipdb``. Thanks, `Tiago Ilieve <https://github.com/myhro>`__. (:pr:`2342`)
 - Fixed a confusing error that occurred if ``metadata.json`` contained nested objects. (:issue:`2403`)
 - Fixed a bug with ``?_trace=1`` where it returned a blank page if the response was larger than 256KB. (:issue:`2404`)
 - Tracing mechanism now also displays SQL queries that returned errors or ran out of time. `datasette-pretty-traces 0.5 <https://github.com/simonw/datasette-pretty-traces/releases/tag/0.5>`__ includes support for displaying this new type of trace. (:issue:`2405`)
@@ -148,7 +407,7 @@ This release focuses on performance, in particular against large tables, and int
 - Failed CSRF checks now display a more user-friendly error page. (:issue:`2390`)
 - Fixed a bug where the ``json1`` extension was not correctly detected on the ``/-/versions`` page. Thanks, `Seb Bacon <https://github.com/sebbacon>`__. (:issue:`2326`)
 - Fixed a bug where the Datasette write API did not correctly accept ``Content-Type: application/json; charset=utf-8``. (:issue:`2384`)
-- Fixed a bug where Datasette would fail to start if ``metadata.yml`` contained a ``queries`` block. (`#2386 <https://github.com/simonw/datasette/pull/2386>`__)
+- Fixed a bug where Datasette would fail to start if ``metadata.yml`` contained a ``queries`` block. (:pr:`2386`)
 
 .. _v1_0_a14:
 
@@ -161,13 +420,13 @@ This alpha introduces significant changes to Datasette's :ref:`metadata` system,
 - Metadata about tables, databases, instances and columns is now stored in :ref:`internals_internal`. Thanks, Alex Garcia. (:issue:`2341`)
 - Database write connections now execute using the ``IMMEDIATE`` isolation level for SQLite. This should help avoid a rare ``SQLITE_BUSY`` error that could occur when a transaction upgraded to a write mid-flight. (:issue:`2358`)
 - Fix for a bug where canned queries with named parameters could fail against SQLite 3.46. (:issue:`2353`)
-- Datasette now serves ``E-Tag`` headers for static files. Thanks, `Agustin Bacigalup <https://github.com/redraw>`__. (`#2306 <https://github.com/simonw/datasette/pull/2306>`__)
+- Datasette now serves ``E-Tag`` headers for static files. Thanks, `Agustin Bacigalup <https://github.com/redraw>`__. (:pr:`2306`)
 - Dropdown menus now use a ``z-index`` that should avoid them being hidden by plugins. (:issue:`2311`)
 - Incorrect table and row names are no longer reflected back on the resulting 404 page. (:issue:`2359`)
 - Improved documentation for async usage of the :ref:`plugin_hook_track_event` hook. (:issue:`2319`)
 - Fixed some HTTPX deprecation warnings. (:issue:`2307`)
 - Datasette now serves a ``<html lang="en">`` attribute. Thanks, `Charles Nepote <https://github.com/CharlesNepote>`__. (:issue:`2348`)
-- Datasette's automated tests now run against the maximum and minimum supported versions of SQLite: 3.25 (from September 2018) and 3.46 (from May 2024). Thanks, Alex Garcia. (`#2352 <https://github.com/simonw/datasette/pull/2352>`__)
+- Datasette's automated tests now run against the maximum and minimum supported versions of SQLite: 3.25 (from September 2018) and 3.46 (from May 2024). Thanks, Alex Garcia. (:pr:`2352`)
 - Fixed an issue where clicking twice on the URL output by ``datasette --root`` produced a confusing error. (:issue:`2375`)
 
 .. _v0_64_8:
@@ -262,7 +521,7 @@ To avoid similar mistakes in the future the ``datasette.permission_allowed()`` m
 Permission checks now consider opinions from every plugin
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The ``datasette.permission_allowed()`` method previously consulted every plugin that implemented the :ref:`permission_allowed() <plugin_hook_permission_allowed>` plugin hook and obeyed the opinion of the last plugin to return a value. (:issue:`2275`)
+The ``datasette.permission_allowed()`` method previously consulted every plugin that implemented the ``permission_allowed()`` plugin hook and obeyed the opinion of the last plugin to return a value. (:issue:`2275`)
 
 Datasette now consults every plugin and checks to see if any of them returned ``False`` (the veto rule), and if none of them did, it then checks to see if any of them returned ``True``.
 
@@ -308,7 +567,7 @@ Configuration
 - The ``-s/--setting`` option can now be used to set plugin configuration as well. See :ref:`configuration_cli` for details. (:issue:`2252`)
 
   The above YAML configuration example using ``-s/--setting`` looks like this:
-  
+
   .. code-block:: bash
 
         datasette mydatabase.db \
@@ -331,7 +590,7 @@ This provides two initial hooks, with more to come in the future:
 - :ref:`makeAboveTablePanelConfigs() <javascript_plugins_makeAboveTablePanelConfigs>` can add additional panels to the top of the table page.
 - :ref:`makeColumnActions() <javascript_plugins_makeColumnActions>` can add additional actions to the column menu.
 
-Thanks `Cameron Yick <https://github.com/hydrosquall>`__ for contributing this feature. (`#2052 <https://github.com/simonw/datasette/pull/2052>`__)
+Thanks `Cameron Yick <https://github.com/hydrosquall>`__ for contributing this feature. (:pr:`2052`)
 
 Plugin hooks
 ~~~~~~~~~~~~
@@ -397,7 +656,7 @@ Minor fixes
 - Datasette now checks if the user has permission to view a table linked to by a foreign key before turning that foreign key into a clickable link. (:issue:`2178`)
 - The ``execute-sql`` permission now implies that the actor can also view the database and instance. (:issue:`2169`)
 - Documentation describing a pattern for building plugins that themselves :ref:`define further hooks <writing_plugins_extra_hooks>` for other plugins. (:issue:`1765`)
-- Datasette is now tested against the Python 3.12 preview. (`#2175 <https://github.com/simonw/datasette/pull/2175>`__)
+- Datasette is now tested against the Python 3.12 preview. (:pr:`2175`)
 
 .. _v1_0_a5:
 
@@ -422,7 +681,7 @@ For more information and workarounds, read `the security advisory <https://githu
 Also in this alpha:
 
 - The new ``datasette plugins --requirements`` option outputs a list of currently installed plugins in Python ``requirements.txt`` format, useful for duplicating that installation elsewhere. (:issue:`2133`)
-- :ref:`canned_queries_writable` can now define a ``on_success_message_sql`` field in their configuration, containing a SQL query that should be executed upon successful completion of the write operation in order to generate a message to be shown to the user. (:issue:`2138`)
+- :ref:`queries_writable` can now define a ``on_success_message_sql`` field in their configuration, containing a SQL query that should be executed upon successful completion of the write operation in order to generate a message to be shown to the user. (:issue:`2138`)
 - The automatically generated border color for a database is now shown in more places around the application. (:issue:`2119`)
 - Every instance of example shell script code in the documentation should now include a working copy button, free from additional syntax. (:issue:`2140`)
 
@@ -600,11 +859,11 @@ Features
 ~~~~~~~~
 
 - Now tested against Python 3.11. Docker containers used by ``datasette publish`` and ``datasette package`` both now use that version of Python. (:issue:`1853`)
-- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (`#1789 <https://github.com/simonw/datasette/pull/1789>`__)
+- ``--load-extension`` option now supports entrypoints. Thanks, Alex Garcia. (:pr:`1789`)
 - Facet size can now be set per-table with the new ``facet_size`` table metadata option. (:issue:`1804`)
 - The :ref:`setting_truncate_cells_html` setting now also affects long URLs in columns. (:issue:`1805`)
 - The non-JavaScript SQL editor textarea now increases height to fit the SQL query. (:issue:`1786`)
-- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (`#1794 <https://github.com/simonw/datasette/pull/1794>`__)
+- Facets are now displayed with better line-breaks in long values. Thanks, Daniel Rech. (:pr:`1794`)
 - The ``settings.json`` file used in :ref:`config_dir` is now validated on startup. (:issue:`1816`)
 - SQL queries can now include leading SQL comments, using ``/* ... */`` or ``-- ...`` syntax. Thanks,  Charles Nepote. (:issue:`1860`)
 - SQL query is now re-displayed when terminated with a time limit error. (:issue:`1819`)
@@ -626,7 +885,7 @@ Documentation
 - New tutorial: `Cleaning data with sqlite-utils and Datasette <https://datasette.io/tutorials/clean-data>`__.
 - Screenshots in the documentation are now maintained using `shot-scraper <https://shot-scraper.datasette.io/>`__, as described in `Automating screenshots for the Datasette documentation using shot-scraper <https://simonwillison.net/2022/Oct/14/automating-screenshots/>`__. (:issue:`1844`)
 - More detailed command descriptions on the :ref:`CLI reference <cli_reference>` page. (:issue:`1787`)
-- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (`#1825 <https://github.com/simonw/datasette/pull/1825>`__)
+- New documentation on :ref:`deploying_openrc` - thanks, Adam Simpson. (:pr:`1825`)
 
 .. _v0_62:
 
@@ -642,14 +901,14 @@ Features
 
 - Datasette is now compatible with `Pyodide <https://pyodide.org/>`__.  This is the enabling technology behind `Datasette Lite <https://lite.datasette.io/>`__. (:issue:`1733`)
 - Database file downloads now implement conditional GET using ETags. (:issue:`1739`)
-- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (`#1759 <https://github.com/simonw/datasette/pull/1759>`__)
+- HTML for facet results and suggested results has been extracted out into new templates ``_facet_results.html`` and ``_suggested_facets.html``. Thanks, M. Nasimul Haque. (:pr:`1759`)
 - Datasette now runs some SQL queries in parallel. This has limited impact on performance, see `this research issue <https://github.com/simonw/datasette/issues/1727>`__ for details.
 - New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`)
 - Spaces in the database names in URLs are now encoded as ``+`` rather than ``~20``. (:issue:`1701`)
 - ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`)
 - The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image <https://hub.docker.com/datasetteproject/datasette>`__ has been upgraded to ``3.10.6-slim-bullseye``.  (:issue:`1768`)
 - Canned writable queries against immutable databases now show a warning message. (:issue:`1728`)
-- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (`#1717 <https://github.com/simonw/datasette/pull/1717>`__)
+- ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (:pr:`1717`)
 - ``datasette publish cloudrun`` has new ``--min-instances`` and ``--max-instances`` options. (:issue:`1779`)
 
 Plugin hooks
@@ -657,7 +916,7 @@ Plugin hooks
 
 - New plugin hook: :ref:`handle_exception() <plugin_hook_handle_exception>`, for custom handling of exceptions caught by Datasette. (:issue:`1770`)
 - The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook is now also passed a ``row`` argument, representing the ``sqlite3.Row`` object that is being rendered. (:issue:`1300`)
-- The :ref:`configuration directory <config_dir>` is now stored in ``datasette.config_dir``, making it available to plugins. Thanks, Chris Amico. (`#1766 <https://github.com/simonw/datasette/pull/1766>`__)
+- The :ref:`configuration directory <config_dir>` is now stored in ``datasette.config_dir``, making it available to plugins. Thanks, Chris Amico. (:pr:`1766`)
 
 Bug fixes
 ~~~~~~~~~
@@ -710,7 +969,7 @@ Datasette also now requires Python 3.7 or higher.
 - Common Datasette symbols can now be imported directly from the top-level ``datasette`` package, see :ref:`internals_shortcuts`. Those symbols are ``Response``, ``Forbidden``, ``NotFound``, ``hookimpl``, ``actor_matches_allow``. (:issue:`957`)
 - ``/-/versions`` page now returns additional details for libraries used by SpatiaLite. (:issue:`1607`)
 - Documentation now links to the `Datasette Tutorials <https://datasette.io/tutorials>`__.
-- Datasette will now also look for SpatiaLite in ``/opt/homebrew`` - thanks, Dan Peterson. (`#1649 <https://github.com/simonw/datasette/pull/1649>`__)
+- Datasette will now also look for SpatiaLite in ``/opt/homebrew`` - thanks, Dan Peterson. (:pr:`1649`)
 - Fixed bug where :ref:`custom pages <custom_pages>` did not work on Windows. Thanks, Robert Christie. (:issue:`1545`)
 - Fixed error caused when a table had a column named ``n``. (:issue:`1228`)
 
@@ -809,14 +1068,14 @@ Other small fixes
 - New :ref:`register_commands() <plugin_hook_register_commands>` plugin hook allows plugins to register additional Datasette CLI commands, e.g. ``datasette mycommand file.db``. (:issue:`1449`)
 - Adding ``?_facet_size=max`` to a table page now shows the number of unique values in each facet. (:issue:`1423`)
 - Upgraded dependency `httpx 0.20 <https://github.com/encode/httpx/releases/tag/0.20.0>`__ - the undocumented ``allow_redirects=`` parameter to :ref:`internals_datasette_client` is now ``follow_redirects=``, and defaults to ``False`` where it previously defaulted to ``True``. (:issue:`1488`)
-- The ``--cors`` option now causes Datasette to return the ``Access-Control-Allow-Headers: Authorization`` header, in addition to ``Access-Control-Allow-Origin: *``. (`#1467 <https://github.com/simonw/datasette/pull/1467>`__)
+- The ``--cors`` option now causes Datasette to return the ``Access-Control-Allow-Headers: Authorization`` header, in addition to ``Access-Control-Allow-Origin: *``. (:pr:`1467`)
 - Code that figures out which named parameters a SQL query takes in order to display form fields for them is no longer confused by strings that contain colon characters. (:issue:`1421`)
 - Renamed ``--help-config`` option to ``--help-settings``. (:issue:`1431`)
 - ``datasette.databases`` property is now a documented API. (:issue:`1443`)
 - The ``base.html`` template now wraps everything other than the ``<footer>`` in a ``<div class="not-footer">`` element, to help with advanced CSS customization. (:issue:`1446`)
 - The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook can now return an awaitable function. This means the hook can execute SQL queries. (:issue:`1425`)
 - :ref:`plugin_register_routes` plugin hook now accepts an optional ``datasette`` argument. (:issue:`1404`)
-- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`canned_queries_options`. (:issue:`1422`)
+- New ``hide_sql`` canned query option for defaulting to hiding the SQL query used by a canned query, see :ref:`queries_options`. (:issue:`1422`)
 - New ``--cpu`` option for :ref:`datasette publish cloudrun <publish_cloud_run>`. (:issue:`1420`)
 - If `Rich <https://github.com/willmcgugan/rich>`__ is installed in the same virtual environment as Datasette, it will be used to provide enhanced display of error tracebacks on the console. (:issue:`1416`)
 - ``datasette.utils`` :ref:`internals_utils_parse_metadata` function, used by the new `datasette-remote-metadata plugin <https://datasette.io/plugins/datasette-remote-metadata>`__, is now a documented API. (:issue:`1405`)
@@ -838,7 +1097,7 @@ Other small fixes
 - New ``datasette --uds /tmp/datasette.sock`` option for binding Datasette to a Unix domain socket, see :ref:`proxy documentation <deploying_proxy>` (:issue:`1388`)
 - ``"searchmode": "raw"`` table metadata option for defaulting a table to executing SQLite full-text search syntax without first escaping it, see :ref:`full_text_search_advanced_queries`. (:issue:`1389`)
 - New plugin hook: ``get_metadata()``, for returning custom metadata for an instance, database or table. Thanks, Brandon Roberts! (:issue:`1384`)
-- New plugin hook: :ref:`plugin_hook_skip_csrf`, for opting out of CSRF protection based on the incoming request. (:issue:`1377`)
+- New plugin hook: ``skip_csrf``, for opting out of CSRF protection based on the incoming request. (:issue:`1377`)
 - The :ref:`menu_links() <plugin_hook_menu_links>`, :ref:`table_actions() <plugin_hook_table_actions>` and :ref:`database_actions() <plugin_hook_database_actions>` plugin hooks all gained a new optional ``request`` argument providing access to the current request. (:issue:`1371`)
 - Major performance improvement for Datasette faceting. (:issue:`1394`)
 - Improved documentation for :ref:`deploying_proxy` to recommend using ``ProxyPreservehost On`` with Apache. (:issue:`1387`)
@@ -908,8 +1167,8 @@ Documentation improvements, bug fixes and support for SpatiaLite 5.
 - The :ref:`Response.asgi_send() <internals_response_asgi_send>` method is now documented. (:issue:`1266`)
 - The official Datasette Docker image now bundles SpatiaLite version 5. (:issue:`1278`)
 - Fixed a ``no such table: pragma_database_list`` bug when running Datasette against SQLite versions prior to SQLite 3.16.0. (:issue:`1276`)
-- HTML lists displayed in table cells are now styled correctly. Thanks, Bob Whitelock. (:issue:`1141`, `#1252 <https://github.com/simonw/datasette/pull/1252>`__)
-- Configuration directory mode now correctly serves immutable databases that are listed in ``inspect-data.json``. Thanks Campbell Allen and Frankie Robertson. (`#1031 <https://github.com/simonw/datasette/pull/1031>`__, `#1229 <https://github.com/simonw/datasette/pull/1229>`__)
+- HTML lists displayed in table cells are now styled correctly. Thanks, Bob Whitelock. (:issue:`1141`, :pr:`1252`)
+- Configuration directory mode now correctly serves immutable databases that are listed in ``inspect-data.json``. Thanks Campbell Allen and Frankie Robertson. (:pr:`1031`, :pr:`1229`)
 
 .. _v0_55:
 
@@ -1086,7 +1345,7 @@ A new visual design, plugin hooks for adding navigation options, better handling
 New visual design
 ~~~~~~~~~~~~~~~~~
 
-Datasette is no longer white and grey with blue and purple links! `Natalie Downe <https://twitter.com/natbat>`__ has been working on a visual refresh, the first iteration of which is included in this release. (`#1056 <https://github.com/simonw/datasette/pull/1056>`__)
+Datasette is no longer white and grey with blue and purple links! `Natalie Downe <https://twitter.com/natbat>`__ has been working on a visual refresh, the first iteration of which is included in this release. (:pr:`1056`)
 
 .. image:: datasette-0.51.png
    :width: 740px
@@ -1163,7 +1422,7 @@ New :ref:`deploying` documentation with guides for deploying Datasette on a Linu
 
 Other improvements in this release:
 
-- :ref:`publish_cloud_run` documentation now covers Google Cloud SDK options. Thanks, Geoffrey Hing. (`#995 <https://github.com/simonw/datasette/pull/995>`__)
+- :ref:`publish_cloud_run` documentation now covers Google Cloud SDK options. Thanks, Geoffrey Hing. (:pr:`995`)
 - New ``datasette -o`` option which opens your browser as soon as Datasette starts up. (:issue:`970`)
 - Datasette now sets ``sqlite3.enable_callback_tracebacks(True)`` so that errors in custom SQL functions will display tracebacks. (:issue:`891`)
 - Fixed two rendering bugs with column headers in portrait mobile view. (:issue:`978`, :issue:`980`)
@@ -1190,7 +1449,7 @@ See also `Datasette 0.50: The annotated release notes <https://simonwillison.net
 
 See also `Datasette 0.49: The annotated release notes <https://simonwillison.net/2020/Sep/15/datasette-0-49/>`__.
 
-- Writable canned queries now expose a JSON API, see :ref:`canned_queries_json_api`. (:issue:`880`)
+- Writable canned queries now expose a JSON API, see :ref:`queries_json_api`. (:issue:`880`)
 - New mechanism for defining page templates with custom path parameters - a template file called ``pages/about/{slug}.html`` will be used to render any requests to ``/about/something``. See :ref:`custom_pages_parameters`. (:issue:`944`)
 - ``register_output_renderer()`` render functions can now return a ``Response``. (:issue:`953`)
 - New ``--upgrade`` option for ``datasette install``. (:issue:`945`)
@@ -1267,7 +1526,7 @@ See also `Datasette 0.49: The annotated release notes <https://simonwillison.net
 - ``tests`` are now excluded from the Datasette package properly - thanks, abeyerpath. (:issue:`456`)
 - The Datasette package published to PyPI now includes ``sdist`` as well as ``bdist_wheel``.
 - Better titles for canned query pages. (:issue:`887`)
-- Now only loads Python files from a directory passed using the ``--plugins-dir`` option - thanks, Amjith Ramanujam. (`#890 <https://github.com/simonw/datasette/pull/890>`__)
+- Now only loads Python files from a directory passed using the ``--plugins-dir`` option - thanks, Amjith Ramanujam. (:pr:`890`)
 - New documentation section on :ref:`publish_vercel`.
 
 .. _v0_45:
@@ -1282,7 +1541,7 @@ Magic parameters for canned queries, a log out feature, improved plugin document
 Magic parameters for canned queries
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Canned queries now support :ref:`canned_queries_magic_parameters`, which can be used to insert or select automatically generated values. For example::
+Canned queries now support :ref:`queries_magic_parameters`, which can be used to insert or select automatically generated values. For example::
 
     insert into logs
       (user_id, timestamp)
@@ -1313,7 +1572,7 @@ New plugin hooks
 
 - :ref:`plugin_hook_register_magic_parameters` can be used to define new types of magic canned query parameters.
 - :ref:`plugin_hook_startup` can run custom code when Datasette first starts up. `datasette-init <https://github.com/simonw/datasette-init>`__ is a new plugin that uses this hook to create database tables and views on startup if they have not yet been created. (:issue:`834`)
-- :ref:`plugin_hook_canned_queries` lets plugins provide additional canned queries beyond those defined in Datasette's metadata. See `datasette-saved-queries <https://github.com/simonw/datasette-saved-queries>`__ for an example of this hook in action. (:issue:`852`)
+- ``canned_queries()`` lets plugins provide additional canned queries beyond those defined in Datasette's metadata. See `datasette-saved-queries <https://github.com/simonw/datasette-saved-queries>`__ for an example of this hook in action. (:issue:`852`)
 - :ref:`plugin_hook_forbidden` is a hook for customizing how Datasette responds to 403 forbidden errors. (:issue:`812`)
 
 Smaller changes
@@ -1381,14 +1640,14 @@ You can use the new ``"allow"`` block syntax in ``metadata.json`` (or ``metadata
 
 See :ref:`authentication_permissions_allow` for more details.
 
-Plugins can implement their own custom permission checks using the new :ref:`plugin_hook_permission_allowed` hook.
+Plugins can implement their own custom permission checks using the new ``plugin_hook_permission_allowed()`` plugin hook.
 
 A new debug page at ``/-/permissions`` shows recent permission checks, to help administrators and plugin authors understand exactly what checks are being performed. This tool defaults to only being available to the root user, but can be exposed to other users by plugins that respond to the ``permissions-debug`` permission. (:issue:`788`)
 
 Writable canned queries
 ~~~~~~~~~~~~~~~~~~~~~~~
 
-Datasette's :ref:`canned_queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example.
+Datasette's :ref:`queries` feature lets you define SQL queries in ``metadata.json`` which can then be executed by users visiting a specific URL. https://latest.datasette.io/fixtures/neighborhood_search for example.
 
 Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 introduces the ability for canned queries to execute ``INSERT`` or ``UPDATE`` queries as well, using the new ``"write": true`` property (:issue:`800`):
 
@@ -1407,7 +1666,7 @@ Canned queries were previously restricted to ``SELECT``, but Datasette 0.44 intr
         }
     }
 
-See :ref:`canned_queries_writable` for more details.
+See :ref:`queries_writable` for more details.
 
 Flash messages
 ~~~~~~~~~~~~~~
@@ -1462,7 +1721,7 @@ Smaller changes
 - New ``request.cookies`` property.
 - ``/-/plugins`` endpoint now shows a list of hooks implemented by each plugin, e.g. https://latest.datasette.io/-/plugins?all=1
 - ``request.post_vars()`` method no longer discards empty values.
-- New "params" canned query key for explicitly setting named parameters, see :ref:`canned_queries_named_parameters`. (:issue:`797`)
+- New "params" canned query key for explicitly setting named parameters, see :ref:`queries_named_parameters`. (:issue:`797`)
 - ``request.args`` is now a :ref:`MultiParams <internals_multiparams>` object.
 - Fixed a bug with the ``datasette plugins`` command. (:issue:`802`)
 - Nicer pattern for using ``make_app_client()`` in tests. (:issue:`395`)
@@ -1495,8 +1754,8 @@ The main focus of this release is a major upgrade to the :ref:`plugin_register_o
 * Redesign of :ref:`plugin_register_output_renderer` to provide more context to the render callback and support an optional ``"can_render"`` callback that controls if a suggested link to the output format is provided. (:issue:`581`, :issue:`770`)
 * Visually distinguish float and integer columns - useful for figuring out why order-by-column might be returning unexpected results. (:issue:`729`)
 * The :ref:`internals_request`, which is passed to several plugin hooks, is now documented. (:issue:`706`)
-* New ``metadata.json`` option for setting a custom default page size for specific tables and views, see :ref:`metadata_page_size`. (:issue:`751`)
-* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega <https://github.com/simonw/datasette-vega>`__, see :ref:`canned_queries_options`. (:issue:`706`)
+* New ``metadata.json`` option for setting a custom default page size for specific tables and views, see :ref:`table_configuration_size`. (:issue:`751`)
+* Canned queries can now be configured with a default URL fragment hash, useful when working with plugins such as `datasette-vega <https://github.com/simonw/datasette-vega>`__, see :ref:`queries_options`. (:issue:`706`)
 * Fixed a bug in ``datasette publish`` when running on operating systems where the ``/tmp`` directory lives in a different volume, using a backport of the Python 3.8 ``shutil.copytree()`` function. (:issue:`744`)
 * Every plugin hook is now covered by the unit tests, and a new unit test checks that each plugin hook has at least one corresponding test. (:issue:`771`, :issue:`773`)
 
@@ -1536,7 +1795,7 @@ Also in this release:
 * Datasette now has a *pattern portfolio* at ``/-/patterns`` - e.g. https://latest.datasette.io/-/patterns. This is a page that shows every Datasette user interface component in one place, to aid core development and people building custom CSS themes. (:issue:`151`)
 * SQLite `PRAGMA functions <https://www.sqlite.org/pragma.html#pragfunc>`__ such as ``pragma_table_info(tablename)`` are now allowed in Datasette SQL queries. (:issue:`761`)
 * Datasette pages now consistently return a ``content-type`` of ``text/html; charset=utf-8"``. (:issue:`752`)
-* Datasette now handles an ASGI ``raw_path`` value of ``None``, which should allow compatibility with the `Mangum <https://github.com/erm/mangum>`__ adapter for running ASGI apps on AWS Lambda. Thanks, Colin Dellow. (`#719 <https://github.com/simonw/datasette/pull/719>`__)
+* Datasette now handles an ASGI ``raw_path`` value of ``None``, which should allow compatibility with the `Mangum <https://github.com/erm/mangum>`__ adapter for running ASGI apps on AWS Lambda. Thanks, Colin Dellow. (:pr:`719`)
 * Installation documentation now covers how to :ref:`installation_pipx`. (:issue:`756`)
 * Improved the documentation for :ref:`full_text_search`. (:issue:`748`)
 
@@ -1559,7 +1818,7 @@ Also in this release:
 -----------------
 
 * New :ref:`setting_base_url` configuration setting for serving up the correct links while running Datasette under a different URL prefix. (:issue:`394`)
-* New metadata settings ``"sort"`` and ``"sort_desc"`` for setting the default sort order for a table. See :ref:`metadata_default_sort`. (:issue:`702`)
+* New metadata settings ``"sort"`` and ``"sort_desc"`` for setting the default sort order for a table. See :ref:`table_configuration_sort`. (:issue:`702`)
 * Sort direction arrow now displays by default on the primary key. This means you only have to click once (not twice) to sort in reverse order. (:issue:`677`)
 * New ``await Request(scope, receive).post_vars()`` method for accessing POST form variables. (:issue:`700`)
 * :ref:`plugin_hooks` documentation now links to example uses of each plugin. (:issue:`709`)
@@ -1589,7 +1848,7 @@ Also in this release:
 -----------------
 
 * Plugins now have a supported mechanism for writing to a database, using the new ``.execute_write()`` and ``.execute_write_fn()`` methods. :ref:`Documentation <database_execute_write>`. (:issue:`682`)
-* Immutable databases that have had their rows counted using the ``inspect`` command now use the calculated count more effectively - thanks, Kevin Keogh. (`#666 <https://github.com/simonw/datasette/pull/666>`__)
+* Immutable databases that have had their rows counted using the ``inspect`` command now use the calculated count more effectively - thanks, Kevin Keogh. (:pr:`666`)
 * ``--reload`` no longer restarts the server if a database file is modified, unless that database was opened immutable mode with ``-i``. (:issue:`494`)
 * New ``?_searchmode=raw`` option turns off escaping for FTS queries in ``?_search=`` allowing full use of SQLite's `FTS5 query syntax <https://www.sqlite.org/fts5.html#full_text_query_syntax>`__. (:issue:`676`)
 
@@ -1610,7 +1869,7 @@ Also in this release:
 
 * Added five new plugins and one new conversion tool to the :ref:`ecosystem`.
 * The ``Datasette`` class has a new ``render_template()`` method which can be used by plugins to render templates using Datasette's pre-configured `Jinja <https://jinja.palletsprojects.com/>`__ templating library.
-* You can now execute SQL queries that start with a ``-- comment`` - thanks, Jay Graves (`#653 <https://github.com/simonw/datasette/pull/653>`__)
+* You can now execute SQL queries that start with a ``-- comment`` - thanks, Jay Graves (:pr:`653`)
 
 .. _v0_34:
 
@@ -1618,7 +1877,7 @@ Also in this release:
 -----------------
 
 * ``_search=`` queries are now correctly escaped using a new ``escape_fts()`` custom SQL function. This means you can now run searches for strings like ``park.`` without seeing errors. (:issue:`651`)
-* `Google Cloud Run <https://cloud.google.com/run/>`__ is no longer in beta, so ``datasette publish cloudrun`` has been updated to work even if the user has not installed the ``gcloud`` beta components package. Thanks, Katie McLaughlin (`#660 <https://github.com/simonw/datasette/pull/660>`__)
+* `Google Cloud Run <https://cloud.google.com/run/>`__ is no longer in beta, so ``datasette publish cloudrun`` has been updated to work even if the user has not installed the ``gcloud`` beta components package. Thanks, Katie McLaughlin (:pr:`660`)
 * ``datasette package`` now accepts a ``--port`` option for specifying which port the resulting Docker container should listen on. (:issue:`661`)
 
 .. _v0_33:
@@ -1656,7 +1915,7 @@ Datasette now renders templates using `Jinja async mode <https://jinja.palletspr
 0.31.1 (2019-11-12)
 -------------------
 
-- Deployments created using ``datasette publish``  now use ``python:3.8`` base Docker image (`#629 <https://github.com/simonw/datasette/pull/629>`__)
+- Deployments created using ``datasette publish``  now use ``python:3.8`` base Docker image (:pr:`629`)
 
 .. _v0_31:
 
@@ -1669,10 +1928,10 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you
 
     pip install datasette==0.30.2
 
-- Format SQL button now works with read-only SQL queries - thanks, Tobias Kunze (`#602 <https://github.com/simonw/datasette/pull/602>`__)
+- Format SQL button now works with read-only SQL queries - thanks, Tobias Kunze (:pr:`602`)
 - New ``?column__notin=x,y,z`` filter for table views (:issue:`614`)
 - Table view now uses ``select col1, col2, col3`` instead of ``select *``
-- Database filenames can now contain spaces - thanks, Tobias Kunze (`#590 <https://github.com/simonw/datasette/pull/590>`__)
+- Database filenames can now contain spaces - thanks, Tobias Kunze (:pr:`590`)
 - Removed obsolete ``?_group_count=col`` feature (:issue:`504`)
 - Improved user interface and documentation for ``datasette publish cloudrun`` (:issue:`608`)
 - Tables with indexes now show the ``CREATE INDEX`` statements on the table page (:issue:`618`)
@@ -1708,7 +1967,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you
 - Allow ``EXPLAIN WITH...`` (:issue:`583`)
 - Button to format SQL - thanks, Tobias Kunze (:issue:`136`)
 - Sort databases on homepage by argument order - thanks, Tobias Kunze (:issue:`585`)
-- Display metadata footer on custom SQL queries - thanks, Tobias Kunze (`#589 <https://github.com/simonw/datasette/pull/589>`__)
+- Display metadata footer on custom SQL queries - thanks, Tobias Kunze (:pr:`589`)
 - Use ``--platform=managed`` for ``publish cloudrun`` (:issue:`587`)
 - Fixed bug returning non-ASCII characters in CSV (:issue:`584`)
 - Fix for ``/foo`` v.s. ``/foo-bar`` bug (:issue:`601`)
@@ -1719,7 +1978,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you
 -------------------
 
 - Fixed implementation of CodeMirror on database page (:issue:`560`)
-- Documentation typo fixes - thanks, Min ho Kim (`#561 <https://github.com/simonw/datasette/pull/561>`__)
+- Documentation typo fixes - thanks, Min ho Kim (:pr:`561`)
 - Mechanism for detecting if a table has FTS enabled now works if the table name used alternative escaping mechanisms (:issue:`570`) - for compatibility with `a recent change to sqlite-utils <https://github.com/simonw/sqlite-utils/pull/57>`__.
 
 .. _v0_29_2:
@@ -1864,7 +2123,7 @@ Datasette :ref:`facets` provide an intuitive way to quickly summarize and intera
 
 Facet by array (:issue:`359`) is only available if your SQLite installation provides the ``json1`` extension. Datasette will automatically detect columns that contain JSON arrays of values and offer a faceting interface against those columns - useful for modelling things like tags without needing to break them out into a new table. See :ref:`facet_by_json_array` for more.
 
-The new :ref:`plugin_register_facet_classes` plugin hook (`#445 <https://github.com/simonw/datasette/pull/445>`__) can be used to register additional custom facet classes. Each facet class should provide two methods: ``suggest()`` which suggests facet selections that might be appropriate for a provided SQL query, and ``facet_results()`` which executes a facet operation and returns results. Datasette's own faceting implementations have been refactored to use the same API as these plugins.
+The new :ref:`plugin_register_facet_classes` plugin hook (:pr:`445`) can be used to register additional custom facet classes. Each facet class should provide two methods: ``suggest()`` which suggests facet selections that might be appropriate for a provided SQL query, and ``facet_results()`` which executes a facet operation and returns results. Datasette's own faceting implementations have been refactored to use the same API as these plugins.
 
 .. _v0_28_publish_cloudrun:
 
@@ -1873,7 +2132,7 @@ datasette publish cloudrun
 
 `Google Cloud Run <https://cloud.google.com/run/>`__ is a brand new serverless hosting platform from Google, which allows you to build a Docker container which will run only when HTTP traffic is received and will shut down (and hence cost you nothing) the rest of the time. It's similar to Zeit's Now v1 Docker hosting platform which sadly is `no longer accepting signups <https://hyperion.alpha.spectrum.chat/zeit/now/cannot-create-now-v1-deployments~d206a0d4-5835-4af5-bb5c-a17f0171fb25?m=MTU0Njk2NzgwODM3OA==>`__ from new users.
 
-The new ``datasette publish cloudrun`` command was contributed by Romain Primet (`#434 <https://github.com/simonw/datasette/pull/434>`__) and publishes selected databases to a new Datasette instance running on Google Cloud Run.
+The new ``datasette publish cloudrun`` command was contributed by Romain Primet (:pr:`434`) and publishes selected databases to a new Datasette instance running on Google Cloud Run.
 
 See :ref:`publish_cloud_run` for full documentation.
 
@@ -1882,7 +2141,7 @@ See :ref:`publish_cloud_run` for full documentation.
 register_output_renderer plugins
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Russ Garrett implemented a new Datasette plugin hook called :ref:`register_output_renderer <plugin_register_output_renderer>` (`#441 <https://github.com/simonw/datasette/pull/441>`__) which allows plugins to create additional output renderers in addition to Datasette's default ``.json`` and ``.csv``.
+Russ Garrett implemented a new Datasette plugin hook called :ref:`register_output_renderer <plugin_register_output_renderer>` (:pr:`441`) which allows plugins to create additional output renderers in addition to Datasette's default ``.json`` and ``.csv``.
 
 Russ's in-development `datasette-geo <https://github.com/russss/datasette-geo>`__ plugin includes `an example <https://github.com/russss/datasette-geo/blob/d4cecc020848bbde91e9e17bf352f7c70bc3dccf/datasette_plugin_geo/geojson.py>`__ of this hook being used to output ``.geojson`` automatically converted from SpatiaLite.
 
@@ -1891,7 +2150,7 @@ Russ's in-development `datasette-geo <https://github.com/russss/datasette-geo>`_
 Medium changes
 ~~~~~~~~~~~~~~
 
-- Datasette now conforms to the `Black coding style <https://github.com/python/black>`__ (`#449 <https://github.com/simonw/datasette/pull/449>`__) - and has a unit test to enforce this in the future
+- Datasette now conforms to the `Black coding style <https://github.com/python/black>`__ (:pr:`449`) - and has a unit test to enforce this in the future
 - New :ref:`json_api_table_arguments`:
    - ``?columnname__in=value1,value2,value3`` filter for executing SQL IN queries against a table, see :ref:`table_arguments` (:issue:`433`)
    - ``?columnname__date=yyyy-mm-dd`` filter which returns rows where the spoecified datetime column falls on the specified date (`583b22a <https://github.com/simonw/datasette/commit/583b22aa28e26c318de0189312350ab2688c90b1>`__)
@@ -1912,17 +2171,17 @@ Small changes
 
 - We now show the size of the database file next to the download link (:issue:`172`)
 - New ``/-/databases`` introspection page shows currently connected databases (:issue:`470`)
-- Binary data is no longer displayed on the table and row pages (`#442 <https://github.com/simonw/datasette/pull/442>`__ - thanks, Russ Garrett)
+- Binary data is no longer displayed on the table and row pages (:pr:`442` - thanks, Russ Garrett)
 - New show/hide SQL links on custom query pages (:issue:`415`)
-- The :ref:`extra_body_script <plugin_hook_extra_body_script>` plugin hook now accepts an optional ``view_name`` argument (`#443 <https://github.com/simonw/datasette/pull/443>`__ - thanks, Russ Garrett)
-- Bumped Jinja2 dependency to 2.10.1 (`#426 <https://github.com/simonw/datasette/pull/426>`__)
+- The :ref:`extra_body_script <plugin_hook_extra_body_script>` plugin hook now accepts an optional ``view_name`` argument (:pr:`443` - thanks, Russ Garrett)
+- Bumped Jinja2 dependency to 2.10.1 (:pr:`426`)
 - All table filters are now documented, and documentation is enforced via unit tests (`2c19a27 <https://github.com/simonw/datasette/commit/2c19a27d15a913e5f3dd443f04067169a6f24634>`__)
 - New project guideline: master should stay shippable at all times! (`31f36e1 <https://github.com/simonw/datasette/commit/31f36e1b97ccc3f4387c80698d018a69798b6228>`__)
 - Fixed a bug where ``sqlite_timelimit()`` occasionally failed to clean up after itself (`bac4e01 <https://github.com/simonw/datasette/commit/bac4e01f40ae7bd19d1eab1fb9349452c18de8f5>`__)
 - We no longer load additional plugins when executing pytest (:issue:`438`)
 - Homepage now links to database views if there are less than five tables in a database (:issue:`373`)
 - The ``--cors`` option is now respected by error pages (:issue:`453`)
-- ``datasette publish heroku`` now uses the ``--include-vcs-ignore`` option, which means it works under Travis CI (`#407 <https://github.com/simonw/datasette/pull/407>`__)
+- ``datasette publish heroku`` now uses the ``--include-vcs-ignore`` option, which means it works under Travis CI (:pr:`407`)
 - ``datasette publish heroku`` now publishes using Python 3.6.8 (`666c374 <https://github.com/simonw/datasette/commit/666c37415a898949fae0437099d62a35b1e9c430>`__)
 - Renamed ``datasette publish now`` to ``datasette publish nowv1`` (:issue:`472`)
 - ``datasette publish nowv1`` now accepts multiple ``--alias`` parameters (`09ef305 <https://github.com/simonw/datasette/commit/09ef305c687399384fe38487c075e8669682deb4>`__)
@@ -1994,7 +2253,7 @@ New plugin hooks, improved database view support and an easier way to use more r
 - New ``render_cell`` plugin hook. Plugins can now customize how values are displayed in the HTML tables produced by Datasette's browsable interface. `datasette-json-html <https://github.com/simonw/datasette-json-html>`__ and `datasette-render-images <https://github.com/simonw/datasette-render-images>`__ are two new plugins that use this hook. :ref:`render_cell documentation <plugin_hook_render_cell>`. Closes :issue:`352`
 - New ``extra_body_script`` plugin hook, enabling plugins to provide additional JavaScript that should be added to the page footer. :ref:`extra_body_script documentation <plugin_hook_extra_body_script>`.
 - ``extra_css_urls`` and ``extra_js_urls`` hooks now take additional optional parameters, allowing them to be more selective about which pages they apply to. :ref:`Documentation <plugin_hook_extra_css_urls>`.
-- You can now use the :ref:`sortable_columns metadata setting <metadata_sortable_columns>` to explicitly enable sort-by-column in the interface for database views, as well as for specific tables.
+- You can now use the :ref:`sortable_columns metadata setting <table_configuration_sortable_columns>` to explicitly enable sort-by-column in the interface for database views, as well as for specific tables.
 - The new ``fts_table`` and ``fts_pk`` metadata settings can now be used to :ref:`explicitly configure full-text search for a table or a view <full_text_search_table_or_view>`, even if that table is not directly coupled to the SQLite FTS feature in the database schema itself.
 - Datasette will now use `pysqlite3 <https://github.com/coleifer/pysqlite3>`__ in place of the standard library ``sqlite3`` module if it has been installed in the current environment. This makes it much easier to run Datasette against a more recent version of SQLite, including the just-released `SQLite 3.25.0 <https://www.sqlite.org/releaselog/3_25_0.html>`__ which adds window function support. More details on how to use this in :issue:`360`
 - New mechanism that allows :ref:`plugin configuration options <plugins_configuration>` to be set using ``metadata.json``.
@@ -2013,7 +2272,7 @@ A number of small new features:
 - Documentation for :ref:`datasette publish and datasette package <publishing>`, closes `#337 <https://github.com/simonw/datasette/issues/337>`_
 - Fixed compatibility with Python 3.7
 - ``datasette publish heroku`` now supports app names via the ``-n`` option, which can also be used to overwrite an existing application [Russ Garrett]
-- Title and description metadata can now be set for :ref:`canned SQL queries <canned_queries>`, closes `#342 <https://github.com/simonw/datasette/issues/342>`_
+- Title and description metadata can now be set for :ref:`canned SQL queries <queries>`, closes `#342 <https://github.com/simonw/datasette/issues/342>`_
 - New ``force_https_on`` config option, fixes ``https://`` API URLs when deploying to Zeit Now - closes `#333 <https://github.com/simonw/datasette/issues/333>`_
 - ``?_json_infinity=1`` query string argument for handling Infinity/-Infinity values in JSON, closes `#332 <https://github.com/simonw/datasette/issues/332>`_
 - URLs displayed in the results of custom SQL queries are now URLified, closes `#298 <https://github.com/simonw/datasette/issues/298>`_
@@ -2079,7 +2338,7 @@ Foreign key expansions
 ~~~~~~~~~~~~~~~~~~~~~~
 
 When Datasette detects a foreign key reference it attempts to resolve a label
-for that reference (automatically or using the :ref:`label_columns` metadata
+for that reference (automatically or using the :ref:`table_configuration_label_column` metadata
 option) so it can display a link to the associated row.
 
 This expansion is now also available for JSON and CSV representations of the

docs/cli-reference.rst

@@ -119,8 +119,10 @@ Once started you can access it at ``http://localhost:8001``
                                       signed cookies
       --root                          Output URL that sets a cookie authenticating
                                       the root user
+      --default-deny                  Deny all permissions by default
       --get TEXT                      Run an HTTP GET request against this path,
                                       print results and exit
+      --headers                       Include HTTP headers in --get output
       --token TEXT                    API token to send with --get requests
       --actor TEXT                    Actor to use for --get requests (JSON string)
       --version-note TEXT             Additional note to show on /-/versions
@@ -488,8 +490,15 @@ See :ref:`publish_cloud_run`.
       --cpu [1|2|4]                   Number of vCPUs to allocate in Cloud Run
       --timeout INTEGER               Build timeout in seconds
       --apt-get-install TEXT          Additional packages to apt-get install
-      --max-instances INTEGER         Maximum Cloud Run instances
+      --max-instances INTEGER         Maximum Cloud Run instances (use 0 to remove
+                                      the limit)  [default: 1]
       --min-instances INTEGER         Minimum Cloud Run instances
+      --artifact-repository TEXT      Artifact Registry repository to store the
+                                      image  [default: datasette]
+      --artifact-region TEXT          Artifact Registry location (region or multi-
+                                      region)  [default: us]
+      --artifact-project TEXT         Project ID for Artifact Registry (defaults to
+                                      the active project)
       --help                          Show this message and exit.