Shorten link text in changelog

Refs #2831, #2832, #2841, #2842, #2843, #2846

.github/actions/setup-sqlite-version/action.yml

@@ -0,0 +1,39 @@
+name: "Setup SQLite version"
+description: "Build and activate a specific SQLite version from its amalgamation archive"
+inputs:
+  version:
+    description: "The SQLite version to install"
+    required: true
+  cflags:
+    description: "CFLAGS to use when compiling SQLite"
+    required: false
+    default: ""
+  skip-activate:
+    description: "Set to true to skip modifying the library path"
+    required: false
+    default: "false"
+  fallback-urls:
+    description: "Whitespace-separated fallback download URLs to try after sqlite.org"
+    required: false
+    default: ""
+outputs:
+  sqlite-location:
+    description: "Directory containing the compiled SQLite library"
+    value: ${{ steps.build.outputs.sqlite-location }}
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      run: mkdir -p "$RUNNER_TEMP/sqlite-versions/downloads"
+    - uses: actions/cache@v6
+      with:
+        path: ${{ runner.temp }}/sqlite-versions/downloads
+        key: setup-sqlite-version-${{ inputs.version }}-amalgamation-v1
+    - id: build
+      shell: bash
+      run: bash "$GITHUB_ACTION_PATH/setup-sqlite-version.sh"
+      env:
+        SQLITE_VERSION: ${{ inputs.version }}
+        SQLITE_CFLAGS: ${{ inputs.cflags }}
+        SQLITE_SKIP_ACTIVATE: ${{ inputs.skip-activate }}
+        SQLITE_EXTRA_FALLBACK_URLS: ${{ inputs.fallback-urls }}

.github/actions/setup-sqlite-version/setup-sqlite-version.sh

@@ -0,0 +1,144 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+version_spec="${SQLITE_VERSION:?SQLITE_VERSION is required}"
+cflags="${SQLITE_CFLAGS:-}"
+skip_activate="${SQLITE_SKIP_ACTIVATE:-false}"
+extra_fallback_urls="${SQLITE_EXTRA_FALLBACK_URLS:-}"
+
+case "$version_spec" in
+  3.46 | 3.46.0)
+    sqlite_version="3.46.0"
+    sqlite_year="2024"
+    amalgamation_id="3460000"
+    builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3460000.zip"
+    ;;
+  3.25 | 3.25.0)
+    sqlite_version="3.25.0"
+    sqlite_year="2018"
+    amalgamation_id="3250000"
+    builtin_fallback_urls="https://static.simonwillison.net/static/2026/sqlite-amalgamation-3250000.zip?v=1"
+    ;;
+  *)
+    echo "::error::Unsupported SQLite version '$version_spec'. Add its release year and amalgamation id to $GITHUB_ACTION_PATH/setup-sqlite-version.sh."
+    exit 1
+    ;;
+esac
+
+case "$(uname -s)" in
+  Linux)
+    library_name="libsqlite3.so.0"
+    library_path_var="LD_LIBRARY_PATH"
+    ;;
+  Darwin)
+    library_name="libsqlite3.dylib"
+    library_path_var="DYLD_LIBRARY_PATH"
+    ;;
+  *)
+    echo "::error::Unsupported platform $(uname -s)"
+    exit 1
+    ;;
+esac
+
+runner_temp="${RUNNER_TEMP:-}"
+if [ -z "$runner_temp" ]; then
+  runner_temp="$(mktemp -d)"
+fi
+
+filename="sqlite-amalgamation-${amalgamation_id}"
+official_url="https://www.sqlite.org/${sqlite_year}/${filename}.zip"
+download_dir="${runner_temp}/sqlite-versions/downloads"
+source_root="${runner_temp}/sqlite-versions/source"
+source_dir="${source_root}/${filename}"
+build_dir="${runner_temp}/sqlite-versions/build/${sqlite_version}"
+archive_path="${download_dir}/${filename}.zip"
+
+mkdir -p "$download_dir" "$source_root" "$build_dir"
+
+download_archive() {
+  local url
+  local candidate_path="${archive_path}.tmp"
+  local urls=("$official_url")
+
+  for url in $builtin_fallback_urls $extra_fallback_urls; do
+    urls+=("$url")
+  done
+
+  rm -f "$candidate_path"
+  for url in "${urls[@]}"; do
+    echo "Downloading SQLite ${sqlite_version} amalgamation from ${url}"
+    if curl \
+      --fail \
+      --location \
+      --show-error \
+      --retry 5 \
+      --retry-delay 2 \
+      --retry-max-time 180 \
+      --retry-all-errors \
+      --connect-timeout 20 \
+      --max-time 240 \
+      --output "$candidate_path" \
+      "$url"; then
+      mv "$candidate_path" "$archive_path"
+      return 0
+    fi
+
+    echo "::warning::Download failed from ${url}"
+    rm -f "$candidate_path"
+  done
+
+  echo "::error::Could not download SQLite ${sqlite_version} amalgamation"
+  return 1
+}
+
+if [ ! -f "${source_dir}/sqlite3.c" ]; then
+  if [ ! -f "$archive_path" ]; then
+    download_archive
+  fi
+
+  rm -rf "$source_dir"
+  unzip -q "$archive_path" -d "$source_root"
+fi
+
+if [ ! -f "${source_dir}/sqlite3.c" ]; then
+  echo "::error::Expected ${source_dir}/sqlite3.c after extracting ${archive_path}"
+  exit 1
+fi
+
+read -r -a cflag_args <<< "$cflags"
+
+echo "Compiling SQLite ${sqlite_version} to ${build_dir}/${library_name}"
+gcc \
+  -fPIC \
+  -shared \
+  "${cflag_args[@]}" \
+  "${source_dir}/sqlite3.c" \
+  "-I${source_dir}" \
+  -o "${build_dir}/${library_name}"
+
+if [ "$library_name" = "libsqlite3.so.0" ]; then
+  ln -sf "$library_name" "${build_dir}/libsqlite3.so"
+fi
+
+if [ -n "${GITHUB_OUTPUT:-}" ]; then
+  echo "sqlite-location=${build_dir}" >> "$GITHUB_OUTPUT"
+else
+  echo "sqlite-location=${build_dir}"
+fi
+
+case "$(printf '%s' "$skip_activate" | tr '[:upper:]' '[:lower:]')" in
+  true | 1 | yes)
+    echo "Skipping ${library_path_var} activation"
+    ;;
+  *)
+    existing_value="${!library_path_var:-}"
+    if [ -n "${GITHUB_ENV:-}" ]; then
+      if [ -n "$existing_value" ]; then
+        echo "${library_path_var}=${build_dir}:${existing_value}" >> "$GITHUB_ENV"
+      else
+        echo "${library_path_var}=${build_dir}" >> "$GITHUB_ENV"
+      fi
+    fi
+    echo "Added ${build_dir} to ${library_path_var}"
+    ;;
+esac

.github/workflows/deploy-latest.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out datasette
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
     - name: Set up Python
       uses: actions/setup-python@v6
       with:

.github/workflows/documentation-links.yml

@@ -1,16 +0,0 @@
-name: Read the Docs Pull Request Preview
-on:
-  pull_request:
-    types:
-      - opened
-
-permissions:
-  pull-requests: write
-
-jobs:
-  documentation-links:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: readthedocs/actions/preview@v1
-        with:
-          project-slug: "datasette"

.github/workflows/playwright.yml

@@ -0,0 +1,48 @@
+name: Playwright
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: [chromium, firefox, webkit]
+    steps:
+    - uses: actions/checkout@v7
+    - name: Set up Python 3.14
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.14"
+        allow-prereleases: true
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Cache uv
+      uses: actions/cache@v6
+      with:
+        path: ~/.cache/uv
+        key: ${{ runner.os }}-py3.14-uv-${{ hashFiles('pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-py3.14-uv-
+    - name: Cache Playwright browsers
+      uses: actions/cache@v6
+      with:
+        path: ~/.cache/ms-playwright/
+        key: ${{ runner.os }}-playwright-${{ matrix.browser }}-${{ hashFiles('pyproject.toml') }}
+        restore-keys: |
+          ${{ runner.os }}-playwright-${{ matrix.browser }}-
+    - name: Install uv
+      run: python -m pip install uv
+    - name: Install dependencies
+      run: uv sync --group dev --group playwright
+    - name: Install ${{ matrix.browser }}
+      run: uv run --group dev --group playwright playwright install --with-deps ${{ matrix.browser }}
+    - name: Run Playwright tests
+      run: uv run --group dev --group playwright pytest tests/test_playwright.py --playwright --browser ${{ matrix.browser }}

.github/workflows/prettier.yml

@@ -10,8 +10,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out repo
-      uses: actions/checkout@v6
-    - uses: actions/cache@v5
+      uses: actions/checkout@v7
+    - uses: actions/cache@v6
       name: Configure npm caching
       with:
         path: ~/.npm

.github/workflows/publish.yml

@@ -14,7 +14,7 @@ jobs:
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
       with:
@@ -35,7 +35,7 @@ jobs:
     permissions:
       id-token: write
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -56,7 +56,7 @@ jobs:
     needs: [deploy]
     if: "!github.event.release.prerelease"
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -92,7 +92,7 @@ jobs:
     needs: [deploy]
     if: "!github.event.release.prerelease"
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Build and push to Docker Hub
       env:
         DOCKER_USER: ${{ secrets.DOCKER_USER }}

.github/workflows/push_docker_tag.yml

@@ -13,7 +13,7 @@ jobs:
   deploy_docker:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Build and push to Docker Hub
       env:
         DOCKER_USER: ${{ secrets.DOCKER_USER }}

.github/workflows/spellcheck.yml

@@ -9,7 +9,7 @@ jobs:
   spellcheck:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Set up Python
       uses: actions/setup-python@v6
       with:

.github/workflows/stable-docs.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
       with:
         fetch-depth: 0  # We need all commits to find docs/ changes
     - name: Set up Git user

.github/workflows/test-coverage.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out datasette
-      uses: actions/checkout@v6
+      uses: actions/checkout@v7
     - name: Set up Python
       uses: actions/setup-python@v6
       with:

.github/workflows/test-pyodide.yml

@@ -12,7 +12,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Set up Python 3.10
       uses: actions/setup-python@v6
       with:
@@ -20,7 +20,7 @@ jobs:
         cache: 'pip'
         cache-dependency-path: '**/pyproject.toml'
     - name: Cache Playwright browsers
-      uses: actions/cache@v5
+      uses: actions/cache@v6
       with:
         path: ~/.cache/ms-playwright/
         key: ${{ runner.os }}-browsers

.github/workflows/test-sqlite-support.yml

@@ -25,7 +25,7 @@ jobs:
           #"3.23.1" # 2018-04-10, before UPSERT
         ]
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
       with:
@@ -34,7 +34,7 @@ jobs:
         cache: pip
         cache-dependency-path: pyproject.toml
     - name: Set up SQLite ${{ matrix.sqlite-version }}
-      uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6
+      uses: ./.github/actions/setup-sqlite-version
       with:
         version: ${{ matrix.sqlite-version }}
         cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1"

.github/workflows/test.yml

@@ -9,10 +9,11 @@ jobs:
   test:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v6
       with:

.github/workflows/tmate-mac.yml

@@ -10,6 +10,6 @@ jobs:
   build:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Setup tmate session
       uses: mxschmitt/action-tmate@v3

.github/workflows/tmate.yml

@@ -11,7 +11,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v7
     - name: Setup tmate session
       uses: mxschmitt/action-tmate@v3
       env:

.gitignore

@@ -1,8 +1,12 @@
 build-metadata.json
 datasets.json
 
+.playwright-mcp
+
 scratchpad
 
+ignored/
+
 .vscode
 
 uv.lock
@@ -131,4 +135,4 @@ tests/*.dylib
 tests/*.so
 tests/*.dll
 
-.idea
+.idea

Justfile

@@ -11,16 +11,33 @@ export DATASETTE_SECRET := "not_a_secret"
 @test *options: init
   uv run pytest -n auto {{options}}
 
+# Install Playwright browser support, Chromium by default
+@playwright-install browser="chromium":
+  uv run --group playwright playwright install {{browser}}
+
+# Install all Playwright browsers used by the test suite
+@playwright-install-all:
+  uv run --group playwright playwright install chromium firefox webkit
+
+# Run Playwright tests, Chromium by default
+@playwright browser="chromium" *options:
+  uv run --group playwright pytest tests/test_playwright.py --playwright --browser {{browser}} {{options}}
+
+# Run Playwright tests against all supported browsers
+@playwright-all *options:
+  uv run --group playwright pytest tests/test_playwright.py --playwright --browser chromium --browser firefox --browser webkit {{options}}
+
 @codespell:
   uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt
   uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt
   uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt
   uv run codespell tests --ignore-words docs/codespell-ignore-words.txt
 
-# Run linters: black, ruff, cog
+# Run linters: black, ruff, prettier, cog
 @lint: codespell
   uv run black datasette tests --check
   uv run ruff check datasette tests
+  npm run prettier -- --check
   uv run cog --check README.md docs/*.rst
 
 # Apply ruff fixes

datasette/__init__.py

@@ -1,8 +1,14 @@
 from datasette.permissions import Permission  # noqa
 from datasette.version import __version_info__, __version__  # noqa
 from datasette.events import Event  # noqa
-from datasette.tokens import TokenHandler, TokenRestrictions  # noqa
-from datasette.utils.asgi import Forbidden, NotFound, Request, Response  # noqa
+from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions  # noqa
+from datasette.utils.asgi import (  # noqa
+    Forbidden,
+    NotFound,
+    PayloadTooLarge,
+    Request,
+    Response,
+)
 from datasette.utils import actor_matches_allow  # noqa
 from datasette.views import Context  # noqa
 from .hookspecs import hookimpl  # noqa

datasette/app.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import contextvars
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Sequence
 
 if TYPE_CHECKING:
     from datasette.permissions import Resource
@@ -12,7 +12,6 @@ import dataclasses
 import datetime
 import functools
 import glob
-import hashlib
 import httpx
 import importlib.metadata
 import inspect
@@ -35,6 +34,7 @@ from jinja2 import (
     ChoiceLoader,
     Environment,
     FileSystemLoader,
+    pass_context,
     PrefixLoader,
 )
 from jinja2.environment import Template
@@ -47,9 +47,14 @@ from .views import Context
 from .views.database import (
     database_download,
     DatabaseView,
-    TableCreateView,
     QueryView,
 )
+from .views.table_create_alter import (
+    DatabaseForeignKeyTargetsView,
+    TableAlterView,
+    TableCreateView,
+    TableForeignKeySuggestionsView,
+)
 from .views.execute_write import ExecuteWriteAnalyzeView, ExecuteWriteView
 from .views.stored_queries import (
     QueryCreateAnalyzeView,
@@ -66,6 +71,7 @@ from .views.index import IndexView
 from .views.special import (
     JsonDataView,
     PatternPortfolioView,
+    AutocompleteDebugView,
     AuthTokenView,
     ApiExplorerView,
     CreateTokenView,
@@ -82,10 +88,12 @@ from .views.special import (
     TableSchemaView,
 )
 from .views.table import (
+    TableAutocompleteView,
     TableInsertView,
     TableUpsertView,
     TableSetColumnTypeView,
     TableDropView,
+    TableFragmentView,
     table_view,
 )
 from .views.row import RowView, RowDeleteView, RowUpdateView
@@ -103,6 +111,7 @@ from .utils import (
     baseconv,
     call_with_supported_arguments,
     detect_json1,
+    add_cors_headers,
     display_actor,
     escape_css_string,
     escape_sqlite,
@@ -114,6 +123,7 @@ from .utils import (
     parse_metadata,
     resolve_env_secrets,
     resolve_routes,
+    sha256_file,
     tilde_decode,
     tilde_encode,
     to_css_class,
@@ -121,8 +131,10 @@ from .utils import (
     redact_keys,
     row_sql_params_pks,
 )
+from .tokens import TokenInvalid
 from .utils.asgi import (
     AsgiLifespan,
+    BadRequest,
     Forbidden,
     NotFound,
     DatabaseNotFound,
@@ -197,6 +209,11 @@ SETTINGS = (
         100,
         "Maximum rows that can be inserted at a time using the bulk insert API",
     ),
+    Setting(
+        "max_post_body_bytes",
+        2 * 1024 * 1024,
+        "Maximum size in bytes for a POST body read into memory, e.g. JSON API requests - set 0 to disable this limit",
+    ),
     Setting(
         "num_sql_threads",
         3,
@@ -291,12 +308,21 @@ DEFAULT_NOT_SET = object()
 ResourcesSQL = collections.namedtuple("ResourcesSQL", ("sql", "params"))
 
 
+def _permission_cache_key(actor, action, parent, child):
+    # Key on the full serialized actor so actors differing in any field
+    # (e.g. token restrictions) never share cache entries
+    actor_key = (
+        json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None
+    )
+    return (actor_key, action, parent, child)
+
+
 async def favicon(request, send):
     await asgi_send_file(
         send,
         str(FAVICON_PATH),
         content_type="image/png",
-        headers={"Cache-Control": "max-age=3600, immutable, public"},
+        headers={"Cache-Control": "max-age=3600, public"},
     )
 
 
@@ -313,6 +339,57 @@ def _to_string(value):
         return json.dumps(value, default=str)
 
 
+def _template_context_json_default(value):
+    if dataclasses.is_dataclass(value) and not isinstance(value, type):
+        return {
+            field.name: getattr(value, field.name)
+            for field in dataclasses.fields(value)
+        }
+    return repr(value)
+
+
+@pass_context
+def _legacy_template_csrftoken(context):
+    request = context.get("request")
+    if request and "csrftoken" in request.scope:
+        return request.scope["csrftoken"]()
+    return ""
+
+
+def _resolve_static_asset_path(root_path, path):
+    root = Path(root_path).resolve()
+    full_path = (root / path).resolve()
+    try:
+        full_path.relative_to(root)
+    except ValueError:
+        raise ValueError("Static asset path cannot escape static root") from None
+    return full_path
+
+
+# Documentation for the variables Datasette.render_template() adds to the
+# context for every page. This is part of the documented template contract:
+# keys added in render_template() must be documented here - the contract
+# tests in tests/test_template_context.py enforce this, and the docs in
+# docs/template_context.rst are generated from it.
+TEMPLATE_BASE_CONTEXT = {
+    "request": "The current :ref:`Request object <internals_request>`, or None. Common properties include ``request.path``, ``request.args``, ``request.actor``, ``request.url_vars`` and ``request.host``.",
+    "crumb_items": 'Async function returning breadcrumb navigation items for the current page. Call it with ``request=request`` plus optional ``database=`` and ``table=`` arguments; it returns a list of ``{"href": url, "label": label}`` dictionaries.',
+    "urls": "Object with methods for constructing URLs within Datasette. Common methods include ``urls.instance()``, ``urls.database(database)``, ``urls.table(database, table)``, ``urls.query(database, query)``, ``urls.row(database, table, row_path)`` and ``urls.static(path)`` - see :ref:`internals_datasette_urls`.",
+    "actor": "The currently authenticated actor dictionary, or None. Actors usually include an ``id`` key and may include any other keys supplied by authentication plugins.",
+    "menu_links": "Async function returning links for the Datasette application menu, including links added by plugins. Each item is a link dictionary with ``href`` and ``label`` keys. See :ref:`plugin_hook_menu_links`; for page action menus that can also include JavaScript-backed buttons, see :ref:`plugin_actions`.",
+    "display_actor": "Function that accepts an actor dictionary and returns the display string used in the navigation menu.",
+    "show_logout": "True if the logout link should be shown in the navigation menu",
+    "zip": "Python's ``zip()`` builtin, made available to template logic",
+    "body_scripts": 'List of JavaScript snippets contributed by plugins using :ref:`plugin_hook_extra_body_script`. Each item is a dictionary with ``script`` containing JavaScript source and ``module`` indicating whether Datasette will wrap it in ``<script type="module">``; otherwise Datasette wraps it in a regular ``<script>`` block.',
+    "format_bytes": "Function that accepts a byte count integer and returns a human-readable string such as ``1.2 MB``.",
+    "show_messages": "Function returning any messages set for the current user, clearing them in the process. Returns a list of ``(message, type)`` pairs, where ``type`` is one of Datasette's ``INFO``, ``WARNING`` or ``ERROR`` constants.",
+    "extra_css_urls": "List of extra CSS stylesheets to include on the page. Each item is a dictionary with ``url`` and optional ``sri`` keys, from plugins and configuration.",
+    "extra_js_urls": "List of extra JavaScript URLs to include on the page. Each item is a dictionary with ``url`` plus optional ``sri`` and ``module`` keys, from plugins and configuration.",
+    "base_url": "The configured :ref:`setting_base_url` setting",
+    "datasette_version": "The version of Datasette that is running",
+}
+
+
 class Datasette:
     # Message constants:
     INFO = 1
@@ -405,6 +482,7 @@ class Datasette:
         self._internal_database.name = INTERNAL_DB_NAME
 
         self.cache_headers = cache_headers
+        self._static_asset_hashes = {}
         self.cors = cors
         config_files = []
         metadata_files = []
@@ -545,6 +623,8 @@ class Datasette:
         )
         environment.filters["escape_css_string"] = escape_css_string
         environment.filters["quote_plus"] = urllib.parse.quote_plus
+        environment.globals["csrftoken"] = _legacy_template_csrftoken
+        environment.globals["static"] = self.static
         self._jinja_env = environment
         environment.filters["escape_sqlite"] = escape_sqlite
         environment.filters["to_css_class"] = to_css_class
@@ -613,9 +693,12 @@ class Datasette:
                 return action
         return None
 
-    async def refresh_schemas(self):
+    async def refresh_schemas(self, *, force=False):
         # Throttle schema refreshes to at most once per second
-        if time.monotonic() - getattr(self, "_last_schema_refresh", 0) < 1.0:
+        if (
+            not force
+            and time.monotonic() - getattr(self, "_last_schema_refresh", 0) < 1.0
+        ):
             return
         self._last_schema_refresh = time.monotonic()
         if self._refresh_schemas_lock.locked():
@@ -670,19 +753,7 @@ class Datasette:
             # Compare schema versions to see if we should skip it
             if schema_version == current_schema_versions.get(database_name):
                 continue
-            placeholders = "(?, ?, ?, ?)"
-            values = [database_name, str(db.path), db.is_memory, schema_version]
-            if db.path is None:
-                placeholders = "(?, null, ?, ?)"
-                values = [database_name, db.is_memory, schema_version]
-            await internal_db.execute_write(
-                """
-                INSERT OR REPLACE INTO catalog_databases (database_name, path, is_memory, schema_version)
-                VALUES {}
-            """.format(placeholders),
-                values,
-            )
-            await populate_schema_tables(internal_db, db)
+            await populate_schema_tables(internal_db, db, schema_version)
 
     @property
     def urls(self):
@@ -830,7 +901,9 @@ class Datasette:
         Verify an API token by trying all registered token handlers.
 
         Returns an actor dict from the first handler that recognizes the
-        token, or None if no handler accepts it.
+        token, or None if no handler accepts it. A handler may raise
+        TokenInvalid for a token it recognizes but rejects (bad signature,
+        expired) - Datasette turns that into a 401 response.
         """
         for token_handler in self._token_handlers():
             result = await token_handler.verify_token(self, token)
@@ -1415,24 +1488,55 @@ class Datasette:
 
         return db_plugin_config
 
-    def static_hash(self, filename):
-        if not hasattr(self, "_static_hashes"):
-            self._static_hashes = {}
-        path = os.path.join(str(app_root), "datasette/static", filename)
-        signature = (os.path.getmtime(path), os.path.getsize(path))
-        cached = self._static_hashes.get(filename)
-        if cached and cached["signature"] == signature:
-            return cached["hash"]
-        with open(path) as fp:
-            static_hash = hashlib.sha1(fp.read().encode("utf8")).hexdigest()[:6]
-        self._static_hashes[filename] = {
-            "signature": signature,
-            "hash": static_hash,
-        }
-        return static_hash
+    def _static_asset_path(self, path):
+        return _resolve_static_asset_path(app_root / "datasette" / "static", path)
 
-    def app_css_hash(self):
-        return self.static_hash("app.css")
+    def _static_plugin_asset_path(self, plugin_name, path):
+        for plugin in get_plugins():
+            if not plugin["static_path"]:
+                continue
+            possible_names = {plugin["name"], plugin["name"].replace("-", "_")}
+            if plugin_name in possible_names:
+                return _resolve_static_asset_path(plugin["static_path"], path)
+        raise FileNotFoundError(
+            "No static assets found for plugin {}".format(plugin_name)
+        )
+
+    def _static_mounted_asset(self, mount_name, path):
+        mount_name = mount_name.strip("/")
+        for mount, dirname in self.static_mounts:
+            if mount.strip("/") == mount_name:
+                return (
+                    _resolve_static_asset_path(dirname, path),
+                    self.urls.path("/{}/{}".format(mount_name, path.lstrip("/"))),
+                )
+        raise FileNotFoundError("No static mount found for {}".format(mount_name))
+
+    def _static_asset_hash(self, filepath):
+        filepath = Path(filepath)
+        if self.cache_headers:
+            cached = self._static_asset_hashes.get(filepath)
+            if cached:
+                return cached
+        digest = sha256_file(filepath)[:12]
+        if self.cache_headers:
+            self._static_asset_hashes[filepath] = digest
+        return digest
+
+    def static(self, path, plugin=None, mount=None):
+        if plugin and mount:
+            raise ValueError("Use either plugin= or mount=, not both")
+        if plugin:
+            filepath = self._static_plugin_asset_path(plugin, path)
+            url = self.urls.static_plugins(plugin, path)
+        elif mount:
+            filepath, url = self._static_mounted_asset(mount, path)
+        else:
+            filepath = self._static_asset_path(path)
+            url = self.urls.static(path)
+        hash_value = self._static_asset_hash(filepath)
+        separator = "&" if "?" in url else "?"
+        return url + separator + urllib.parse.urlencode({"_hash": hash_value})
 
     def _prepare_connection(self, conn, database):
         conn.row_factory = sqlite3.Row
@@ -1817,46 +1921,124 @@ class Datasette:
             # For global actions, resource can be omitted:
             can_debug = await datasette.allowed(action="permissions-debug", actor=actor)
         """
-        from datasette.utils.actions_sql import check_permission_for_resource
+        results = await self.allowed_many(
+            actions=[action], resource=resource, actor=actor
+        )
+        return results[action]
 
-        # For global actions, resource remains None
+    async def allowed_many(
+        self,
+        *,
+        actions: Sequence[str],
+        resource: "Resource" = None,
+        actor: dict | None = None,
+    ) -> dict[str, bool]:
+        """
+        Check several actions against one resource for one actor.
 
-        # Check if this action has also_requires - if so, check that action first
-        action_obj = self.actions.get(action)
-        if action_obj and action_obj.also_requires:
-            # Must have the required action first
-            if not await self.allowed(
-                action=action_obj.also_requires,
-                resource=resource,
+        Resolves every action (plus any also_requires dependencies) with a
+        single internal database query, instead of one or two queries per
+        action. Results are stored in the request-scoped permission cache,
+        so subsequent datasette.allowed() calls for the same checks within
+        the same request are served from the cache.
+
+        Example:
+            from datasette.resources import TableResource
+            results = await datasette.allowed_many(
+                actions=["edit-schema", "drop-table", "insert-row"],
+                resource=TableResource(database="data", table="exercise"),
                 actor=actor,
-            ):
-                return False
+            )
+            # {"edit-schema": True, "drop-table": True, "insert-row": False}
+        """
+        from datasette.utils.actions_sql import check_permissions_for_actions
+        from datasette.permissions import (
+            _permission_check_cache,
+            _skip_permission_checks,
+        )
 
         # For global actions, resource is None
         parent = resource.parent if resource else None
         child = resource.child if resource else None
 
-        result = await check_permission_for_resource(
-            datasette=self,
-            actor=actor,
-            action=action,
-            parent=parent,
-            child=child,
-        )
+        # Expand also_requires dependencies (transitively) so that each
+        # dependency is resolved within the same batch
+        expanded = []
 
-        # Log the permission check for debugging
-        self._permission_checks.append(
-            PermissionCheck(
-                when=datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        def add_action(name):
+            if name in expanded:
+                return
+            action_obj = self.actions.get(name)
+            if action_obj is None:
+                raise ValueError(f"Unknown action: {name}")
+            expanded.append(name)
+            if action_obj.also_requires:
+                add_action(action_obj.also_requires)
+
+        requested = list(dict.fromkeys(actions))
+        for name in requested:
+            add_action(name)
+
+        # Consult the request-scoped cache, unless permission checks are
+        # being skipped (skip-mode verdicts must never be cached)
+        skip = _skip_permission_checks.get()
+        cache = None if skip else _permission_check_cache.get()
+
+        final = {}
+        to_check = []
+        for name in expanded:
+            if cache is not None:
+                key = _permission_cache_key(actor, name, parent, child)
+                if key in cache:
+                    final[name] = cache[key]
+                    continue
+            to_check.append(name)
+
+        raw = {}
+        if to_check:
+            raw = await check_permissions_for_actions(
+                datasette=self,
                 actor=actor,
-                action=action,
+                actions=to_check,
                 parent=parent,
                 child=child,
-                result=result,
             )
-        )
 
-        return result
+        def resolve(name):
+            # final verdict = own rules AND verdict of also_requires chain
+            if name in final:
+                return final[name]
+            result = raw[name]
+            action_obj = self.actions.get(name)
+            if result and action_obj.also_requires:
+                result = resolve(action_obj.also_requires)
+            final[name] = result
+            return result
+
+        for name in expanded:
+            resolve(name)
+
+        # Cache the freshly computed checks
+        if cache is not None:
+            for name in to_check:
+                cache[_permission_cache_key(actor, name, parent, child)] = final[name]
+
+        # Log every check (including cache hits) for the debug page,
+        # dependencies before the actions that required them
+        when = datetime.datetime.now(datetime.timezone.utc).isoformat()
+        for name in reversed(expanded):
+            self._permission_checks.append(
+                PermissionCheck(
+                    when=when,
+                    actor=actor,
+                    action=name,
+                    parent=parent,
+                    child=child,
+                    result=final[name],
+                )
+            )
+
+        return {name: final[name] for name in requested}
 
     async def ensure_permission(
         self,
@@ -1929,6 +2111,11 @@ class Datasette:
 
         other_table = fk["other_table"]
         other_column = fk["other_column"]
+        if other_column is None:
+            other_pks = await db.primary_keys(other_table)
+            if len(other_pks) != 1:
+                return {}
+            other_column = other_pks[0]
         visible, _ = await self.check_visibility(
             actor,
             action="view-table",
@@ -1979,6 +2166,18 @@ class Datasette:
             for name, d in self.databases.items()
         ]
 
+    async def _connected_databases_for_actor(self, actor):
+        page = await self.allowed_resources("view-database", actor)
+        allowed_names = {resource.parent async for resource in page.all()}
+        return [
+            database
+            for database in self._connected_databases()
+            if database["name"] in allowed_names
+        ]
+
+    async def _databases_data(self, request):
+        return {"databases": await self._connected_databases_for_actor(request.actor)}
+
     def _versions(self):
         conn = sqlite3.connect(":memory:")
         self._prepare_connection(conn, "_memory")
@@ -2162,7 +2361,11 @@ class Datasette:
                 templates = [templates]
             template = self.get_jinja_environment(request).select_template(templates)
         if dataclasses.is_dataclass(context):
-            context = dataclasses.asdict(context)
+            # Shallow conversion - asdict() would deep-copy values, which
+            # is wasteful and fails on values like sqlite3.Row
+            context = {
+                f.name: getattr(context, f.name) for f in dataclasses.fields(context)
+            }
         body_scripts = []
         # pylint: disable=no-member
         for extra_script in pm.hook.extra_body_script(
@@ -2212,6 +2415,8 @@ class Datasette:
                     links.extend(extra_links)
             return links
 
+        # Keys added here must be documented in TEMPLATE_BASE_CONTEXT -
+        # the contract tests fail otherwise
         template_context = {
             **context,
             **{
@@ -2224,7 +2429,6 @@ class Datasette:
                 "show_logout": request is not None
                 and "ds_actor" in request.cookies
                 and request.actor,
-                "app_css_hash": self.app_css_hash(),
                 "zip": zip,
                 "body_scripts": body_scripts,
                 "format_bytes": format_bytes,
@@ -2236,18 +2440,19 @@ class Datasette:
                     "extra_js_urls", template, context, request, view_name
                 ),
                 "base_url": self.setting("base_url"),
-                "csrftoken": (
-                    request.scope["csrftoken"]
-                    if request and "csrftoken" in request.scope
-                    else lambda: ""
-                ),
                 "datasette_version": __version__,
             },
             **extra_template_vars,
         }
         if request and request.args.get("_context") and self.setting("template_debug"):
             return "<pre>{}</pre>".format(
-                escape(json.dumps(template_context, default=repr, indent=4))
+                escape(
+                    json.dumps(
+                        template_context,
+                        default=_template_context_json_default,
+                        indent=4,
+                    )
+                )
             )
 
         return await template.render_async(template_context)
@@ -2319,10 +2524,9 @@ class Datasette:
         def add_route(view, regex):
             routes.append((regex, view))
 
-        add_route(IndexView.as_view(self), r"/(\.(?P<format>jsono?))?$")
-        add_route(IndexView.as_view(self), r"/-/(\.(?P<format>jsono?))?$")
+        add_route(IndexView.as_view(self), r"/(\.(?P<format>json))?$")
+        add_route(IndexView.as_view(self), r"/-/(\.(?P<format>json))?$")
         add_route(permanent_redirect("/-/"), r"/-$")
-        # TODO: /favicon.ico and /-/static/ deserve far-future cache expires
         add_route(favicon, "/favicon.ico")
 
         add_route(
@@ -2357,7 +2561,10 @@ class Datasette:
         )
         add_route(
             JsonDataView.as_view(
-                self, "plugins.json", self._plugins, needs_request=True
+                self,
+                "plugins.json",
+                self._plugins,
+                needs_request=True,
             ),
             r"/-/plugins(\.(?P<format>json))?$",
         )
@@ -2370,11 +2577,18 @@ class Datasette:
             r"/-/config(\.(?P<format>json))?$",
         )
         add_route(
-            JsonDataView.as_view(self, "threads.json", self._threads),
+            JsonDataView.as_view(
+                self, "threads.json", self._threads, permission="permissions-debug"
+            ),
             r"/-/threads(\.(?P<format>json))?$",
         )
         add_route(
-            JsonDataView.as_view(self, "databases.json", self._connected_databases),
+            JsonDataView.as_view(
+                self,
+                "databases.json",
+                self._databases_data,
+                needs_request=True,
+            ),
             r"/-/databases(\.(?P<format>json))?$",
         )
         add_route(
@@ -2387,7 +2601,7 @@ class Datasette:
             JsonDataView.as_view(
                 self,
                 "actions.json",
-                self._actions,
+                lambda: {"actions": self._actions()},
                 template="debug_actions.html",
                 permission="permissions-debug",
             ),
@@ -2449,6 +2663,10 @@ class Datasette:
             wrap_view(PatternPortfolioView, self),
             r"/-/patterns$",
         )
+        add_route(
+            AutocompleteDebugView.as_view(self),
+            r"/-/debug/autocomplete$",
+        )
         add_route(
             wrap_view(database_download, self),
             r"/(?P<database>[^\/\.]+)\.db$",
@@ -2458,6 +2676,10 @@ class Datasette:
             r"/(?P<database>[^\/\.]+)(\.(?P<format>\w+))?$",
         )
         add_route(TableCreateView.as_view(self), r"/(?P<database>[^\/\.]+)/-/create$")
+        add_route(
+            DatabaseForeignKeyTargetsView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/-/foreign-key-targets$",
+        )
         add_route(
             QueryListView.as_view(self),
             r"/(?P<database>[^\/\.]+)/-/queries(\.(?P<format>json))?$",
@@ -2522,10 +2744,26 @@ class Datasette:
             TableUpsertView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/upsert$",
         )
+        add_route(
+            TableAlterView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/alter$",
+        )
+        add_route(
+            TableForeignKeySuggestionsView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/foreign-key-suggestions$",
+        )
         add_route(
             TableSetColumnTypeView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/set-column-type$",
         )
+        add_route(
+            TableFragmentView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/fragment$",
+        )
+        add_route(
+            TableAutocompleteView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/autocomplete$",
+        )
         add_route(
             TableDropView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/drop$",
@@ -2571,6 +2809,10 @@ class Datasette:
         db, table_name, _ = await self.resolve_table(request)
         pk_values = urlsafe_components(request.url_vars["pks"])
         sql, params, pks = await row_sql_params_pks(db, table_name, pk_values)
+        if len(pk_values) != len(pks):
+            raise BadRequest(
+                "URL row identifier does not match the primary key for this table"
+            )
         results = await db.execute(sql, params, truncate=True)
         row = results.first()
         if row is None:
@@ -2612,7 +2854,16 @@ class DatasetteRouter:
         if raw_path:
             path = raw_path.decode("ascii")
         path = path.partition("?")[0]
-        return await self.route_path(scope, receive, send, path)
+        # Give each request a fresh permission check cache, so repeated
+        # datasette.allowed() checks within the request are memoized but
+        # results never persist beyond it
+        from datasette.permissions import _permission_check_cache
+
+        cache_token = _permission_check_cache.set({})
+        try:
+            return await self.route_path(scope, receive, send, path)
+        finally:
+            _permission_check_cache.reset(cache_token)
 
     async def route_path(self, scope, receive, send, path):
         # Strip off base_url if present before routing
@@ -2620,7 +2871,11 @@ class DatasetteRouter:
         if base_url != "/" and path.startswith(base_url):
             path = "/" + path[len(base_url) :]
             scope = dict(scope, route_path=path)
-        request = Request(scope, receive)
+        request = Request(
+            scope,
+            receive,
+            max_post_body_bytes=self.ds.setting("max_post_body_bytes"),
+        )
         # Populate request_messages if ds_messages cookie is present
         try:
             request._messages = self.ds.unsign(
@@ -2640,13 +2895,24 @@ class DatasetteRouter:
         # Handle authentication
         default_actor = scope.get("actor") or None
         actor = None
+        token_error = None
         results = pm.hook.actor_from_request(datasette=self.ds, request=request)
         for result in results:
-            result = await await_me_maybe(result)
+            try:
+                result = await await_me_maybe(result)
+            except TokenInvalid as ex:
+                # A presented token was recognized but rejected - fail the
+                # request with a 401 even if another credential is valid,
+                # but keep awaiting the remaining coroutines first
+                if token_error is None:
+                    token_error = ex
+                continue
             if result and actor is None:
                 actor = result
                 # Don't break — we must await all coroutines to avoid
                 # "coroutine was never awaited" warnings
+        if token_error is not None:
+            return await self.handle_401(request, send, token_error)
         scope_modifications["actor"] = actor or default_actor
         scope = dict(scope, **scope_modifications)
 
@@ -2678,6 +2944,15 @@ class DatasetteRouter:
         except Exception as exception:
             return await self.handle_exception(request, send, exception)
 
+    async def handle_401(self, request, send, exception):
+        # A presented bearer token was recognized by a handler but rejected.
+        # Bearer tokens are API credentials, so this is always JSON.
+        headers = {"www-authenticate": 'Bearer error="invalid_token"'}
+        if self.ds.cors:
+            add_cors_headers(headers)
+        response = Response.error([str(exception)], 401, headers=headers)
+        await response.asgi_send(send)
+
     async def handle_404(self, request, send, exception=None):
         # If path contains % encoding, redirect to tilde encoding
         if "%" in request.path:

datasette/column_types.py

@@ -6,19 +6,17 @@ class SQLiteType(Enum):
     INTEGER = "INTEGER"
     REAL = "REAL"
     BLOB = "BLOB"
-    NULL = "NULL"
+    NUMERIC = "NUMERIC"
 
     @classmethod
-    def from_declared_type(cls, declared_type: str | None) -> "SQLiteType | None":
+    def from_declared_type(cls, declared_type: str | None) -> "SQLiteType":
         if declared_type is None:
-            return cls.NULL
+            return cls.BLOB
 
         normalized = declared_type.strip().upper()
         if not normalized:
-            return cls.NULL
+            return cls.BLOB
 
-        if normalized == cls.NULL.value:
-            return cls.NULL
         if "INT" in normalized:
             return cls.INTEGER
         if any(token in normalized for token in ("CHAR", "CLOB", "TEXT")):
@@ -31,7 +29,7 @@ class SQLiteType(Enum):
         ):
             return cls.REAL
 
-        return None
+        return cls.NUMERIC
 
 
 class ColumnType:

datasette/database.py

@@ -17,6 +17,7 @@ from .utils import (
     detect_fts,
     detect_primary_keys,
     detect_spatialite,
+    escape_sqlite,
     get_all_foreign_keys,
     get_outbound_foreign_keys,
     md5_not_usedforsecurity,
@@ -246,6 +247,7 @@ class Database:
         request=None,
         return_all=False,
         returning_limit=EXECUTE_WRITE_RETURNING_LIMIT,
+        transaction=True,
     ):
         self._check_not_closed()
         if returning_limit < 0:
@@ -258,7 +260,9 @@ class Database:
             )
 
         with trace("sql", database=self.name, sql=sql.strip(), params=params):
-            results = await self.execute_write_fn(_inner, block=block, request=request)
+            results = await self.execute_write_fn(
+                _inner, block=block, request=request, transaction=transaction
+            )
         return results
 
     async def execute_write_script(self, sql, block=True, request=None):
@@ -348,6 +352,7 @@ class Database:
                 self.ds._prepare_connection(self._write_connection, self.name)
             if transaction:
                 with self._write_connection:
+                    self._write_connection.execute("BEGIN IMMEDIATE")
                     result = fn(self._write_connection)
             else:
                 result = fn(self._write_connection)
@@ -477,6 +482,7 @@ class Database:
                 try:
                     if task.transaction:
                         with conn:
+                            conn.execute("BEGIN IMMEDIATE")
                             result = task.fn(conn)
                     else:
                         result = task.fn(conn)
@@ -603,7 +609,7 @@ class Database:
             try:
                 table_count = (
                     await self.execute(
-                        f"select count(*) from (select * from [{table}] limit {self.count_limit + 1})",
+                        f"select count(*) from (select * from {escape_sqlite(table)} limit {self.count_limit + 1})",
                         custom_time_limit=limit,
                     )
                 ).rows[0][0]
@@ -829,10 +835,10 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event):
         # Execute the actual write
         try:
             result = fn(conn)
-        except Exception:
+        except Exception as e:
             # Throw exception into generator so it can handle it
             try:
-                gen.throw(*sys.exc_info())
+                gen.throw(e)
             except StopIteration:
                 pass
             # Re-raise the original exception

datasette/default_actions.py

@@ -61,6 +61,12 @@ def register_actions():
             description="Create tables",
             resource_class=DatabaseResource,
         ),
+        Action(
+            name="create-view",
+            abbr="cv",
+            description="Create views",
+            resource_class=DatabaseResource,
+        ),
         Action(
             name="store-query",
             abbr="sq",
@@ -111,6 +117,12 @@ def register_actions():
             description="Drop tables",
             resource_class=TableResource,
         ),
+        Action(
+            name="drop-view",
+            abbr="dv",
+            description="Drop views",
+            resource_class=TableResource,
+        ),
         # Query-level actions (child-level)
         Action(
             name="view-query",

datasette/default_column_types.py

@@ -76,6 +76,12 @@ class JsonColumnType(ColumnType):
         return None
 
 
+class TextareaColumnType(ColumnType):
+    name = "textarea"
+    description = "Multiline text"
+    sqlite_types = (SQLiteType.TEXT,)
+
+
 @hookimpl
 def register_column_types(datasette):
-    return [UrlColumnType, EmailColumnType, JsonColumnType]
+    return [UrlColumnType, EmailColumnType, JsonColumnType, TextareaColumnType]

datasette/default_debug_menu.py

@@ -37,6 +37,11 @@ DEBUG_MENU_ITEMS = (
         "Debug allow rules",
         "Explore how allow blocks match actors against permission rules.",
     ),
+    (
+        "/-/debug/autocomplete",
+        "Debug autocomplete",
+        "Try out table autocomplete against a detected label column.",
+    ),
     (
         "/-/threads",
         "Debug threads",

datasette/default_permissions/config.py

@@ -96,6 +96,10 @@ class ConfigPermissionProcessor:
         """Evaluate an allow block against the current actor."""
         if allow_block is None:
             return None
+        # Values passed using ``-s permissions.* 1`` or ``0`` are parsed as
+        # integers, but should retain the CLI's boolean 1/0 behavior.
+        if isinstance(allow_block, int) and allow_block in (0, 1):
+            return bool(allow_block)
         return actor_matches_allow(self.actor, allow_block)
 
     def is_in_restriction_allowlist(

datasette/default_table_actions.py

@@ -0,0 +1,29 @@
+from datasette import hookimpl
+from datasette.resources import TableResource
+
+
+@hookimpl
+def table_actions(datasette, actor, database, table, request):
+    async def inner():
+        db = datasette.get_database(database)
+        if not db.is_mutable:
+            return []
+        if not await datasette.allowed(
+            action="alter-table",
+            resource=TableResource(database=database, table=table),
+            actor=actor,
+        ):
+            return []
+        return [
+            {
+                "type": "button",
+                "label": "Alter table",
+                "description": "Change columns and primary key for this table.",
+                "attrs": {
+                    "aria-label": "Alter table {}".format(table),
+                    "data-table-action": "alter-table",
+                },
+            }
+        ]
+
+    return inner

datasette/extras.py

@@ -5,6 +5,8 @@ from typing import ClassVar
 
 from asyncinject import Registry
 
+from datasette.utils.asgi import BadRequest
+
 
 def extra_names_from_request(request):
     extra_bits = request.args.getlist("_extra")
@@ -81,6 +83,16 @@ class ExtraRegistry:
     def public_classes_for_scope(self, scope):
         return self.classes_for_scope(scope, include_internal=False)
 
+    def internal_classes_for_scope(self, scope):
+        # Extras that are available to HTML templates but excluded from
+        # JSON responses - plain Providers are dependency plumbing and
+        # never surface as keys, so they are not included
+        return [
+            cls
+            for cls in self.classes_for_scope(scope)
+            if issubclass(cls, Extra) and not cls.public
+        ]
+
     def _registry_for_scope(self, scope):
         registry = self._scope_registries.get(scope)
         if registry is None:
@@ -103,6 +115,17 @@ class ExtraRegistry:
             self._allowed_names[key] = names
         return names
 
+    def validate_requested(self, requested, scope):
+        """
+        Raise BadRequest if any requested extra name is not a public extra
+        for this scope. Used by data formats such as .json - HTML pages
+        silently ignore unknown names instead.
+        """
+        allowed = self._allowed_names_for_scope(scope, include_internal=False)
+        unknown = sorted(name for name in requested if name not in allowed)
+        if unknown:
+            raise BadRequest("Unknown _extra: {}".format(", ".join(unknown)))
+
     async def resolve(self, requested, context, scope, include_internal=False):
         allowed_names = self._allowed_names_for_scope(scope, include_internal)
         requested_names = [name for name in requested if name in allowed_names]

datasette/facets.py

@@ -85,7 +85,7 @@ class Facet:
         self.database = database
         # For foreign key expansion. Can be None for e.g. stored SQL queries:
         self.table = table
-        self.sql = sql or f"select * from [{table}]"
+        self.sql = sql or f"select * from {escape_sqlite(table)}"
         self.params = params or []
         self.table_config = table_config
         # row_count can be None, in which case we calculate it ourselves:

datasette/forbidden.py

@@ -1,9 +1,19 @@
 from datasette import hookimpl, Response
+from .utils import add_cors_headers
 
 
 @hookimpl(trylast=True)
 def forbidden(datasette, request, message):
     async def inner():
+        if (
+            request.path.split("?")[0].endswith(".json")
+            or "application/json" in (request.headers.get("accept") or "")
+            or request.headers.get("content-type") == "application/json"
+        ):
+            headers = {}
+            if datasette.cors:
+                add_cors_headers(headers)
+            return Response.error(message, 403, headers=headers)
         return Response.html(
             await datasette.render_template(
                 "error.html",

datasette/handle_exception.py

@@ -1,5 +1,5 @@
 from datasette import hookimpl, Response
-from .utils import add_cors_headers
+from .utils import add_cors_headers, error_body
 from .utils.asgi import (
     Base400,
 )
@@ -28,6 +28,7 @@ def handle_exception(datasette, request, exception):
             rich.get_console().print_exception(show_locals=True)
 
         title = None
+        plain_message = None
         if isinstance(exception, Base400):
             status = exception.status
             info = {}
@@ -36,6 +37,7 @@ def handle_exception(datasette, request, exception):
             status = exception.status
             info = exception.error_dict
             message = exception.message
+            plain_message = exception.plain_message
             if exception.message_is_html:
                 message = Markup(message)
             title = exception.title
@@ -45,6 +47,13 @@ def handle_exception(datasette, request, exception):
             message = str(exception)
             traceback.print_exc()
         templates = [f"{status}.html", "error.html"]
+        headers = {}
+        if datasette.cors:
+            add_cors_headers(headers)
+        if request.path.split("?")[0].endswith(".json"):
+            body = dict(info)
+            body.update(error_body(plain_message or message, status))
+            return Response.json(body, status=status, headers=headers)
         info.update(
             {
                 "ok": False,
@@ -53,25 +62,18 @@ def handle_exception(datasette, request, exception):
                 "title": title,
             }
         )
-        headers = {}
-        if datasette.cors:
-            add_cors_headers(headers)
-        if request.path.split("?")[0].endswith(".json"):
-            return Response.json(info, status=status, headers=headers)
-        else:
-            environment = datasette.get_jinja_environment(request)
-            template = environment.select_template(templates)
-            return Response.html(
-                await template.render_async(
-                    dict(
-                        info,
-                        urls=datasette.urls,
-                        app_css_hash=datasette.app_css_hash(),
-                        menu_links=lambda: [],
-                    )
-                ),
-                status=status,
-                headers=headers,
-            )
+        environment = datasette.get_jinja_environment(request)
+        template = environment.select_template(templates)
+        return Response.html(
+            await template.render_async(
+                dict(
+                    info,
+                    urls=datasette.urls,
+                    menu_links=lambda: [],
+                )
+            ),
+            status=status,
+            headers=headers,
+        )
 
     return inner

datasette/hookspecs.py

@@ -159,32 +159,32 @@ def jump_items_sql(datasette, actor, request):
 
 @hookspec
 def row_actions(datasette, actor, request, database, table, row):
-    """Links for the row actions menu"""
+    """Items for the row actions menu"""
 
 
 @hookspec
 def table_actions(datasette, actor, database, table, request):
-    """Links for the table actions menu"""
+    """Items for the table actions menu"""
 
 
 @hookspec
 def view_actions(datasette, actor, database, view, request):
-    """Links for the view actions menu"""
+    """Items for the view actions menu"""
 
 
 @hookspec
 def query_actions(datasette, actor, database, query_name, request, sql, params):
-    """Links for the query and stored query actions menu"""
+    """Items for the query and stored query actions menu"""
 
 
 @hookspec
 def database_actions(datasette, actor, database, request):
-    """Links for the database actions menu"""
+    """Items for the database actions menu"""
 
 
 @hookspec
 def homepage_actions(datasette, actor, request):
-    """Links for the homepage actions menu"""
+    """Items for the homepage actions menu"""
 
 
 @hookspec

datasette/permissions.py

@@ -8,6 +8,14 @@ _skip_permission_checks = contextvars.ContextVar(
     "skip_permission_checks", default=False
 )
 
+# Request-scoped cache of permission check results. The ASGI router sets
+# this to a fresh dict at the start of each request, so cached verdicts
+# never outlive a request or leak between actors. Keys are
+# (actor_json, action, parent, child) tuples, values are booleans.
+_permission_check_cache: contextvars.ContextVar[dict | None] = contextvars.ContextVar(
+    "permission_check_cache", default=None
+)
+
 
 class SkipPermissions:
     """Context manager to temporarily skip permission checks.

datasette/plugins.py

@@ -31,6 +31,7 @@ DEFAULT_PLUGINS = (
     "datasette.default_debug_menu",
     "datasette.default_jump_items",
     "datasette.default_database_actions",
+    "datasette.default_table_actions",
     "datasette.default_query_actions",
     "datasette.handle_exception",
     "datasette.forbidden",

datasette/renderer.py

@@ -1,6 +1,7 @@
 import json
 from datasette.extras import extra_names_from_request
 from datasette.utils import (
+    error_body,
     value_as_boolean,
     remove_infinites,
     CustomJSONEncoder,
@@ -52,8 +53,7 @@ def json_renderer(request, args, data, error, truncated=None):
     if error:
         shape = "objects"
         status_code = 400
-        data["error"] = error
-        data["ok"] = False
+        data.update(error_body(error, status_code))
 
     if truncated is not None:
         data["truncated"] = truncated
@@ -87,7 +87,8 @@ def json_renderer(request, args, data, error, truncated=None):
                         object_rows[pk_string] = row
                     data = object_rows
             if shape_error:
-                data = {"ok": False, "error": shape_error}
+                status_code = 400
+                data = error_body(shape_error, status_code)
         elif shape == "array":
             data = data["rows"]
 
@@ -100,12 +101,7 @@ def json_renderer(request, args, data, error, truncated=None):
             data["rows"] = [list(row.values()) for row in data["rows"]]
     else:
         status_code = 400
-        data = {
-            "ok": False,
-            "error": f"Invalid _shape: {shape}",
-            "status": 400,
-            "title": None,
-        }
+        data = error_body(f"Invalid _shape: {shape}", status_code)
 
     # Don't include "columns" in output
     # https://github.com/simonw/datasette/issues/2136

datasette/static/autocomplete.js

@@ -0,0 +1,344 @@
+(function () {
+  function autocompleteValueFromRow(row) {
+    var pks = (row && row.pks) || {};
+    var keys = Object.keys(pks);
+    if (!keys.length) {
+      return "";
+    }
+    if (keys.length === 1) {
+      return String(pks[keys[0]]);
+    }
+    return keys
+      .map(function (key) {
+        return key + "=" + pks[key];
+      })
+      .join(", ");
+  }
+
+  function autocompleteLabelFromRow(row) {
+    var value = autocompleteValueFromRow(row);
+    if (row.label && String(row.label) !== value) {
+      return row.label + " (" + value + ")";
+    }
+    return value;
+  }
+
+  if (!window.customElements || customElements.get("datasette-autocomplete")) {
+    return;
+  }
+
+  class DatasetteAutocomplete extends HTMLElement {
+    constructor() {
+      super();
+      this.input = null;
+      this.listbox = null;
+      this.status = null;
+      this.results = [];
+      this.activeIndex = -1;
+      this.fetchId = 0;
+      this.searchTimer = null;
+      this.boundInput = this.handleInput.bind(this);
+      this.boundKeydown = this.handleKeydown.bind(this);
+      this.boundBlur = this.handleBlur.bind(this);
+      this.boundFocus = this.handleFocus.bind(this);
+      this.boundPositionListbox = this.positionListbox.bind(this);
+    }
+
+    connectedCallback() {
+      if (this.input) {
+        return;
+      }
+      this.input = this.querySelector("input");
+      if (!this.input) {
+        return;
+      }
+
+      var inputId =
+        this.input.id ||
+        "datasette-autocomplete-" + Math.random().toString(36).slice(2);
+      this.input.id = inputId;
+      var listboxId = inputId + "-listbox";
+      var statusId = inputId + "-status";
+
+      this.classList.add("datasette-autocomplete");
+      this.input.setAttribute("role", "combobox");
+      this.input.setAttribute("aria-autocomplete", "list");
+      this.input.setAttribute("aria-expanded", "false");
+      this.input.setAttribute("aria-controls", listboxId);
+      this.input.setAttribute("autocomplete", "off");
+
+      this.listbox = document.createElement("div");
+      this.listbox.className = "datasette-autocomplete-list";
+      this.listbox.id = listboxId;
+      this.listbox.setAttribute("role", "listbox");
+      this.listbox.hidden = true;
+
+      this.status = document.createElement("span");
+      this.status.className = "datasette-autocomplete-status";
+      this.status.id = statusId;
+      this.status.setAttribute("role", "status");
+      this.status.setAttribute("aria-live", "polite");
+
+      this.input.setAttribute(
+        "aria-describedby",
+        [this.input.getAttribute("aria-describedby"), statusId]
+          .filter(Boolean)
+          .join(" "),
+      );
+
+      this.appendChild(this.listbox);
+      this.appendChild(this.status);
+
+      this.input.addEventListener("input", this.boundInput);
+      this.input.addEventListener("keydown", this.boundKeydown);
+      this.input.addEventListener("blur", this.boundBlur);
+      this.input.addEventListener("focus", this.boundFocus);
+    }
+
+    disconnectedCallback() {
+      if (!this.input) {
+        return;
+      }
+      this.input.removeEventListener("input", this.boundInput);
+      this.input.removeEventListener("keydown", this.boundKeydown);
+      this.input.removeEventListener("blur", this.boundBlur);
+      this.input.removeEventListener("focus", this.boundFocus);
+    }
+
+    handleInput() {
+      this.scheduleSearch();
+    }
+
+    handleFocus() {
+      if (this.input.value.trim() || this.hasAttribute("suggest-on-focus")) {
+        this.scheduleSearch();
+      }
+    }
+
+    handleBlur() {
+      window.setTimeout(() => this.close(), 150);
+    }
+
+    handleKeydown(ev) {
+      if (ev.key === "Escape") {
+        if (!this.listbox.hidden) {
+          ev.preventDefault();
+          this.close();
+        }
+        return;
+      }
+      if (ev.key === "ArrowDown") {
+        ev.preventDefault();
+        if (this.listbox.hidden) {
+          this.scheduleSearch();
+        } else {
+          this.setActiveIndex(this.activeIndex + 1);
+        }
+        return;
+      }
+      if (ev.key === "ArrowUp") {
+        ev.preventDefault();
+        if (!this.listbox.hidden) {
+          this.setActiveIndex(this.activeIndex - 1);
+        }
+        return;
+      }
+      if (ev.key === "Enter" && !this.listbox.hidden && this.activeIndex >= 0) {
+        ev.preventDefault();
+        this.chooseIndex(this.activeIndex);
+      }
+    }
+
+    scheduleSearch() {
+      window.clearTimeout(this.searchTimer);
+      this.searchTimer = window.setTimeout(() => this.search(), 150);
+    }
+
+    async search() {
+      var query = this.input.value.trim();
+      var initial = !query && this.hasAttribute("suggest-on-focus");
+      if (!query && !initial) {
+        this.close();
+        this.status.textContent = "";
+        return;
+      }
+      var src = this.getAttribute("src");
+      if (!src) {
+        return;
+      }
+
+      var url = new URL(src, location.href);
+      url.searchParams.set("q", query);
+      if (initial) {
+        url.searchParams.set("_initial", "1");
+      } else {
+        url.searchParams.delete("_initial");
+      }
+      var fetchId = this.fetchId + 1;
+      this.fetchId = fetchId;
+      this.status.textContent = "Searching...";
+
+      try {
+        var response = await fetch(url.toString(), {
+          headers: {
+            Accept: "application/json",
+          },
+        });
+        if (!response.ok) {
+          throw new Error("HTTP " + response.status);
+        }
+        var data = await response.json();
+        if (fetchId !== this.fetchId) {
+          return;
+        }
+        this.results = (data && data.rows) || [];
+        this.render();
+      } catch (_error) {
+        if (fetchId !== this.fetchId) {
+          return;
+        }
+        this.results = [];
+        this.close();
+        this.status.textContent = "Could not load suggestions";
+      }
+    }
+
+    render() {
+      this.listbox.textContent = "";
+      this.activeIndex = -1;
+      if (!this.results.length) {
+        this.close();
+        this.status.textContent = "No matches";
+        return;
+      }
+
+      this.results.forEach((row, index) => {
+        var option = document.createElement("div");
+        option.className = "datasette-autocomplete-option";
+        option.id = this.input.id + "-option-" + index;
+        option.setAttribute("role", "option");
+        option.setAttribute("aria-selected", "false");
+        option.dataset.index = String(index);
+        option.dataset.value = autocompleteValueFromRow(row);
+        option.textContent = autocompleteLabelFromRow(row);
+        option.addEventListener("mousedown", (ev) => {
+          ev.preventDefault();
+          this.chooseIndex(index);
+        });
+        this.listbox.appendChild(option);
+      });
+
+      this.listbox.hidden = false;
+      this.input.setAttribute("aria-expanded", "true");
+      this.status.textContent =
+        this.results.length + (this.results.length === 1 ? " match" : " matches");
+      this.positionListbox();
+      this.setActiveIndex(0);
+    }
+
+    positionListbox() {
+      if (!this.input || !this.listbox || this.listbox.hidden) {
+        return;
+      }
+
+      var gap = 3;
+      var margin = 8;
+      var inputRect = this.input.getBoundingClientRect();
+      this.listbox.style.maxHeight = "";
+      var defaultMaxHeight = parseFloat(
+        window.getComputedStyle(this.listbox).maxHeight,
+      );
+      if (!Number.isFinite(defaultMaxHeight)) {
+        defaultMaxHeight = 256;
+      }
+      var scrollHeight = Math.ceil(this.listbox.scrollHeight);
+      var desiredHeight = Math.min(scrollHeight, defaultMaxHeight);
+      var availableBelow = Math.max(
+        0,
+        (window.innerHeight || document.documentElement.clientHeight) -
+          inputRect.bottom -
+          gap -
+          margin,
+      );
+
+      this.listbox.style.left = inputRect.left + "px";
+      this.listbox.style.top = inputRect.bottom + gap + "px";
+      this.listbox.style.width = inputRect.width + "px";
+      if (scrollHeight <= defaultMaxHeight && scrollHeight <= availableBelow) {
+        this.listbox.style.maxHeight = "none";
+      } else {
+        this.listbox.style.maxHeight =
+          Math.min(defaultMaxHeight, desiredHeight, availableBelow || defaultMaxHeight) +
+          "px";
+      }
+      window.addEventListener("resize", this.boundPositionListbox);
+      document.addEventListener("scroll", this.boundPositionListbox, true);
+    }
+
+    setActiveIndex(index) {
+      var options = this.listbox.querySelectorAll("[role='option']");
+      if (!options.length) {
+        this.activeIndex = -1;
+        this.input.removeAttribute("aria-activedescendant");
+        return;
+      }
+      if (index < 0) {
+        index = options.length - 1;
+      }
+      if (index >= options.length) {
+        index = 0;
+      }
+      options.forEach((option, optionIndex) => {
+        option.setAttribute(
+          "aria-selected",
+          optionIndex === index ? "true" : "false",
+        );
+      });
+      this.activeIndex = index;
+      this.input.setAttribute("aria-activedescendant", options[index].id);
+    }
+
+    chooseIndex(index) {
+      var row = this.results[index];
+      if (!row) {
+        return;
+      }
+      var value = autocompleteValueFromRow(row);
+      var label = autocompleteLabelFromRow(row);
+      this.input.value = value;
+      this.input.dispatchEvent(new Event("change", { bubbles: true }));
+      this.close();
+      this.status.textContent = "Selected " + label;
+      this.dispatchEvent(
+        new CustomEvent("datasette-autocomplete-select", {
+          bubbles: true,
+          detail: {
+            row: row,
+            value: value,
+            label: label,
+          },
+        }),
+      );
+    }
+
+    close() {
+      if (this.listbox) {
+        this.listbox.hidden = true;
+        this.listbox.textContent = "";
+        this.listbox.style.left = "";
+        this.listbox.style.maxHeight = "";
+        this.listbox.style.top = "";
+        this.listbox.style.width = "";
+      }
+      if (this.input) {
+        this.input.setAttribute("aria-expanded", "false");
+        this.input.removeAttribute("aria-activedescendant");
+      }
+      window.removeEventListener("resize", this.boundPositionListbox);
+      document.removeEventListener("scroll", this.boundPositionListbox, true);
+      this.activeIndex = -1;
+    }
+  }
+
+  customElements.define("datasette-autocomplete", DatasetteAutocomplete);
+})();

datasette/static/column-chooser.js

@@ -31,9 +31,9 @@ class ColumnChooser extends HTMLElement {
       <style>
         :host {
           --ink: #0f0f0f;
-          --paper: #f5f3ef;
+          --paper: #eef6ff;
           --muted: #6b6b6b;
-          --rule: #e2dfd8;
+          --rule: #d8e6f5;
           --accent: #1a56db;
           --accent-light: #e8effd;
           --card: #ffffff;

datasette/static/datasette-manager.js

@@ -82,6 +82,35 @@ const datasetteManager = {
     return columnActions;
   },
 
+  /**
+   * Allows JavaScript plugins to replace or enhance insert/edit modal fields
+   * for specific Datasette column types.
+   *
+   * The first plugin to return a control object wins. Returning null or
+   * undefined means "I do not handle this field".
+   */
+  makeColumnField: (context) => {
+    for (const [pluginName, plugin] of datasetteManager.plugins) {
+      if (!plugin.makeColumnField) {
+        continue;
+      }
+      let control = null;
+      try {
+        control = plugin.makeColumnField(context);
+      } catch (error) {
+        console.error(
+          `Error in makeColumnField() for plugin ${pluginName}`,
+          error,
+        );
+        continue;
+      }
+      if (control) {
+        return Object.assign({ pluginName }, control);
+      }
+    }
+    return null;
+  },
+
   makeJumpSections: (context) => {
     let jumpSections = [];
 

datasette/static/table.js

@@ -12,7 +12,6 @@ var DROPDOWN_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="14" heig
 
 var SET_COLUMN_TYPE_DIALOG_ID = "set-column-type-dialog";
 var setColumnTypeDialogState = null;
-
 function getParams() {
   return new URLSearchParams(location.search);
 }
@@ -634,32 +633,151 @@ const initDatasetteTable = function (manager) {
   });
 };
 
-/* Add x buttons to the filter rows */
-function addButtonsToFilterRows(manager) {
-  var x = "✖";
-  var rows = Array.from(
-    document.querySelectorAll(manager.selectors.filterRow),
+function filterRowSelector(manager) {
+  return manager.selectors.filterRows || manager.selectors.filterRow;
+}
+
+function filterRowsWithControls(manager) {
+  return Array.from(
+    document.querySelectorAll(filterRowSelector(manager)),
   ).filter((el) => el.querySelector(".filter-op"));
-  rows.forEach((row) => {
-    var a = document.createElement("a");
-    a.setAttribute("href", "#");
-    a.setAttribute("aria-label", "Remove this filter");
-    a.style.textDecoration = "none";
-    a.innerText = x;
-    a.addEventListener("click", (ev) => {
-      ev.preventDefault();
-      let row = ev.target.closest("div");
-      row.querySelector("select").value = "";
-      row.querySelector(".filter-op select").value = "exact";
-      row.querySelector("input.filter-value").value = "";
-      ev.target.closest("a").style.display = "none";
-    });
-    row.appendChild(a);
+}
+
+function filterRowNumberFromName(name) {
+  var match = name && name.match(/^_filter_column_(\d+)$/);
+  return match ? parseInt(match[1], 10) : 0;
+}
+
+function nextFilterRowNumber(manager) {
+  return filterRowsWithControls(manager).reduce((max, row) => {
     var column = row.querySelector("select");
-    if (!column.value) {
-      a.style.display = "none";
+    return Math.max(max, filterRowNumberFromName(column && column.name));
+  }, 0) + 1;
+}
+
+function setFilterRowNumber(row, number) {
+  row.querySelector("select").name = `_filter_column_${number}`;
+  row.querySelector(".filter-op select").name = `_filter_op_${number}`;
+  row.querySelector("input.filter-value").name = `_filter_value_${number}`;
+}
+
+function resetFilterRow(row) {
+  row.querySelector("select").value = "";
+  row.querySelector(".filter-op select").value = "exact";
+  row.querySelector("input.filter-value").value = "";
+}
+
+function updateFilterRowButtons(manager) {
+  var rows = filterRowsWithControls(manager);
+  rows.forEach((row, index) => {
+    var removeButton = row.querySelector(".filter-row-remove");
+    var addButton = row.querySelector(".filter-row-add");
+    var column = row.querySelector("select");
+    if (removeButton) {
+      removeButton.hidden = index === 0;
+    }
+    if (addButton) {
+      addButton.hidden = index !== rows.length - 1 || !column.value;
+    }
+    var visibleButtonCount = [removeButton, addButton].filter(function (button) {
+      return button && !button.hidden;
+    }).length;
+    row.classList.toggle(
+      "filter-controls-row-has-buttons",
+      visibleButtonCount > 0,
+    );
+    row.classList.toggle(
+      "filter-controls-row-one-button",
+      visibleButtonCount === 1,
+    );
+    row.classList.toggle(
+      "filter-controls-row-two-buttons",
+      visibleButtonCount === 2,
+    );
+  });
+}
+
+function cloneFilterRow(row) {
+  var clone = row.cloneNode(true);
+  clone.querySelector("select").name = "_filter_column";
+  clone.querySelector(".filter-op select").name = "_filter_op";
+  clone.querySelector("input.filter-value").name = "_filter_value";
+  resetFilterRow(clone);
+  clone.querySelectorAll(".filter-row-icon").forEach((button) => button.remove());
+  return clone;
+}
+
+var FILTER_REMOVE_ICON_SVG = `<svg class="filter-row-remove-icon" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2.1" stroke-linecap="round" stroke-linejoin="round">
+  <path d="M3 6h18"></path>
+  <path d="M8 6V4a2 2 0 0 1 2-2h4a2 2 0 0 1 2 2v2"></path>
+  <path d="M19 6l-1 14a2 2 0 0 1-2 2H8a2 2 0 0 1-2-2L5 6"></path>
+  <path d="M10 11v6"></path>
+  <path d="M14 11v6"></path>
+</svg>`;
+
+var FILTER_ADD_ICON_SVG = `<svg class="filter-row-add-icon" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="16" height="16" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="3" stroke-linecap="round" stroke-linejoin="round">
+  <path d="M5 12h14"></path>
+  <path d="M12 5v14"></path>
+</svg>`;
+
+function addFilterRowButtons(row, manager) {
+  var removeButton = document.createElement("button");
+  removeButton.type = "button";
+  removeButton.className = "filter-row-icon filter-row-remove";
+  removeButton.setAttribute("aria-label", "Remove this filter");
+  removeButton.title = "Remove this filter";
+  removeButton.tabIndex = 0;
+  removeButton.innerHTML = FILTER_REMOVE_ICON_SVG;
+  removeButton.addEventListener("click", (ev) => {
+    var row = ev.currentTarget.closest(filterRowSelector(manager));
+    var rows = filterRowsWithControls(manager);
+    var rowIndex = rows.indexOf(row);
+    var focusRow = rows[rowIndex + 1] || rows[rowIndex - 1] || null;
+    row.remove();
+    updateFilterRowButtons(manager);
+    if (focusRow) {
+      var focusTarget =
+        focusRow.querySelector(".filter-row-add:not([hidden])") ||
+        focusRow.querySelector("select");
+      if (focusTarget) {
+        focusTarget.focus();
+      }
     }
   });
+  row.appendChild(removeButton);
+
+  var addButton = document.createElement("button");
+  addButton.type = "button";
+  addButton.className = "filter-row-icon filter-row-add";
+  addButton.setAttribute("aria-label", "Add another filter");
+  addButton.title = "Add another filter";
+  addButton.tabIndex = 0;
+  addButton.innerHTML = FILTER_ADD_ICON_SVG;
+  addButton.addEventListener("click", (ev) => {
+    var row = ev.currentTarget.closest(filterRowSelector(manager));
+    if (row.querySelector("select").name === "_filter_column") {
+      setFilterRowNumber(row, nextFilterRowNumber(manager));
+    }
+    var clone = cloneFilterRow(row);
+    addFilterRowButtons(clone, manager);
+    row.parentNode.insertBefore(clone, row.nextSibling);
+    updateFilterRowButtons(manager);
+    clone.querySelector("select").focus();
+  });
+  row.appendChild(addButton);
+
+  row.querySelector("select").addEventListener("change", () => {
+    updateFilterRowButtons(manager);
+  });
+}
+
+/* Add buttons to the filter rows */
+function addButtonsToFilterRows(manager) {
+  var rows = filterRowsWithControls(manager);
+  rows.forEach((row) => {
+    addFilterRowButtons(row, manager);
+  });
+  updateFilterRowButtons(manager);
 }
 
 /* Set up datalist autocomplete for filter values */
@@ -688,11 +806,11 @@ function initAutocompleteForFilterValues(manager) {
     });
   }
   createDataLists();
-  // When any select with name=_filter_column changes, update the datalist
+  // When any filter column select changes, update the datalist
   document.body.addEventListener("change", function (event) {
-    if (event.target.name === "_filter_column") {
+    if (event.target.name && event.target.name.startsWith("_filter_column")) {
       event.target
-        .closest(manager.selectors.filterRow)
+        .closest(filterRowSelector(manager))
         .querySelector(".filter-value")
         .setAttribute("list", "datalist-" + event.target.value);
     }

datasette/stored_queries.py

@@ -62,7 +62,6 @@ def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]:
         "description_html": query.description_html,
         "hide_sql": query.hide_sql,
         "fragment": query.fragment,
-        "params": list(query.parameters),
         "parameters": list(query.parameters),
         "is_write": query.is_write,
         "is_private": query.is_private,
@@ -84,7 +83,6 @@ def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]:
     return {
         "queries": [stored_query_to_dict(query) for query in page.queries],
         "next": page.next,
-        "has_more": page.has_more,
         "limit": page.limit,
     }
 

datasette/template_contexts.py

@@ -0,0 +1,40 @@
+"""
+Index of the documented template contexts for Datasette's core HTML pages.
+
+This module deliberately contains no documentation strings of its own -
+the documentation lives next to the code it describes:
+
+- Every page renders a Context dataclass defined in its view module
+  (DatabaseContext, QueryContext in views/database.py, TableContext in
+  views/table.py, RowContext in views/row.py). Fields added by view code
+  carry ``help`` metadata; fields declared with from_extra() take their
+  documentation from the description on the matching Extra class in
+  views/table_extras.py.
+- The keys render_template() adds to every page are documented in
+  TEMPLATE_BASE_CONTEXT in datasette/app.py, next to the code that adds
+  them.
+
+The contract tests in tests/test_template_context.py assert that the real
+rendered context for each page exactly matches what is documented, and
+docs/template_context_doc.py generates docs/template_context.rst from the
+same classes.
+"""
+
+from datasette.app import TEMPLATE_BASE_CONTEXT
+from datasette.views.database import DatabaseContext, QueryContext
+from datasette.views.row import RowContext
+from datasette.views.table import TableContext
+
+PAGES = {
+    "database": DatabaseContext,
+    "query": QueryContext,
+    "table": TableContext,
+    "row": RowContext,
+}
+
+
+def documented_context_keys(page_name):
+    "Set of every documented key for the named page, including base context keys"
+    return set(TEMPLATE_BASE_CONTEXT) | {
+        f.name for f in PAGES[page_name].documented_fields()
+    }

datasette/templates/_action_menu.html

@@ -15,14 +15,22 @@
         <div class="hook"></div>
         <ul role="menu">
             {% for link in action_links %}
-            <li role="none"><a href="{{ link.href }}" role="menuitem" tabindex="-1">{{ link.label }}
-            {% if link.description %}
-                <p class="dropdown-description">{{ link.description }}</p>
-            {% endif %}</a>
+            <li role="none">
+            {% if link.get("type") == "button" %}
+                <button type="button" class="button-as-link action-menu-button" role="menuitem" tabindex="-1"{% for name, value in (link.get("attrs") or {}).items() %} {{ name }}="{{ value }}"{% endfor %}>{{ link.label }}
+                {% if link.description %}
+                    <span class="dropdown-description">{{ link.description }}</span>
+                {% endif %}</button>
+            {% else %}
+                <a href="{{ link.href }}" role="menuitem" tabindex="-1">{{ link.label }}
+                {% if link.description %}
+                    <span class="dropdown-description">{{ link.description }}</span>
+                {% endif %}</a>
+            {% endif %}
             </li>
             {% endfor %}
         </ul>
     </div>
 </details>
 </div>
-{% endif %}
+{% endif %}

datasette/templates/_codemirror.html

@@ -1,5 +1,5 @@
-<script src="{{ base_url }}-/static/sql-formatter-2.3.3.min.js" defer></script>
-<script src="{{ base_url }}-/static/cm-editor-6.0.1.bundle.js"></script>
+<script src="{{ static('sql-formatter-2.3.3.min.js') }}" defer></script>
+<script src="{{ static('cm-editor-6.0.1.bundle.js') }}"></script>
 <style>
   .cm-editor {
     resize: both;

datasette/templates/_permission_ui_styles.html

@@ -6,8 +6,20 @@
     padding: 1.5em;
     margin-bottom: 2em;
 }
+.permission-form form {
+    max-width: 60rem;
+}
+.permission-form-grid {
+    display: grid;
+    gap: 1.5rem;
+    grid-template-columns: repeat(2, minmax(0, 1fr));
+}
+.permission-form-result {
+    margin-top: 1rem;
+    max-width: 60rem;
+}
 .form-section {
-    margin-bottom: 1em;
+    margin-bottom: 1.25em;
 }
 .form-section label {
     display: block;
@@ -15,22 +27,51 @@
     font-weight: bold;
 }
 .form-section input[type="text"],
-.form-section select {
-    width: 100%;
-    max-width: 500px;
-    padding: 0.5em;
+.form-section input[type="number"],
+.form-section select,
+.permission-textarea {
+    background-color: #fff;
+    border: 1px solid #aaa;
+    border-radius: 4px;
     box-sizing: border-box;
-    border: 1px solid #ccc;
-    border-radius: 3px;
+    box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.08);
+    color: #222;
+    font-family: inherit;
+    font-size: 1rem;
+    line-height: 1.4;
+    max-width: none;
+    width: 100%;
+}
+.form-section input[type="text"] {
+    height: 3rem;
+    padding: 0.6rem 0.75rem;
+}
+.form-section input[type="number"] {
+    height: 3rem;
+    max-width: 7rem;
+    padding: 0.6rem 0.75rem;
+}
+.form-section select {
+    height: 3rem;
+    padding: 0.6rem 0.75rem;
+}
+.permission-textarea {
+    font-family: monospace;
+    min-height: 12rem;
+    padding: 0.75rem;
+    resize: vertical;
 }
 .form-section input[type="text"]:focus,
-.form-section select:focus {
-    outline: 2px solid #0066cc;
+.form-section input[type="number"]:focus,
+.form-section select:focus,
+.permission-textarea:focus {
     border-color: #0066cc;
+    box-shadow: 0 0 0 3px rgba(0, 102, 204, 0.18);
+    outline: none;
 }
 .form-section small {
     display: block;
-    margin-top: 0.3em;
+    margin-top: 0.45em;
     color: #666;
 }
 .form-actions {
@@ -142,4 +183,9 @@
     text-align: center;
     color: #666;
 }
+@media only screen and (max-width: 576px) {
+    .permission-form-grid {
+        grid-template-columns: minmax(0, 1fr);
+    }
+}
 </style>

datasette/templates/_permissions_debug_tabs.html

@@ -44,10 +44,10 @@
 </style>
 
 <nav class="permissions-debug-tabs">
-    <a href="{{ urls.path('-/permissions') }}" {% if current_tab == "permissions" %}class="active"{% endif %}>Playground</a>
-    <a href="{{ urls.path('-/check') }}{{ query_string }}" {% if current_tab == "check" %}class="active"{% endif %}>Check</a>
-    <a href="{{ urls.path('-/allowed') }}{{ query_string }}" {% if current_tab == "allowed" %}class="active"{% endif %}>Allowed</a>
-    <a href="{{ urls.path('-/rules') }}{{ query_string }}" {% if current_tab == "rules" %}class="active"{% endif %}>Rules</a>
+    <a href="{{ urls.path('-/check') }}{{ query_string }}" {% if current_tab == "check" %}class="active"{% endif %}>Explain</a>
+    <a href="{{ urls.path('-/allowed') }}{{ query_string }}" {% if current_tab == "allowed" %}class="active"{% endif %}>Access map</a>
+    <a href="{{ urls.path('-/rules') }}{{ query_string }}" {% if current_tab == "rules" %}class="active"{% endif %}>Rule explorer</a>
+    <a href="{{ urls.path('-/permissions') }}" {% if current_tab == "permissions" %}class="active"{% endif %}>Activity</a>
     <a href="{{ urls.path('-/actions') }}" {% if current_tab == "actions" %}class="active"{% endif %}>Actions</a>
     <a href="{{ urls.path('-/allow-debug') }}" {% if current_tab == "allow_debug" %}class="active"{% endif %}>Allow debug</a>
 </nav>

datasette/templates/_table.html

@@ -22,7 +22,7 @@
         </thead>
         <tbody>
         {% for row in display_rows %}
-            <tr>
+            <tr{% if row.pk_path is not none %} data-row="{{ row.row_path }}"{% if row.row_label %} data-row-label="{{ row.row_label }}"{% endif %}{% endif %}>
                 {% for cell in row %}
                     <td class="col-{{ cell.column|to_css_class }} type-{{ cell.value_type }}">{{ cell.value }}</td>
                 {% endfor %}

datasette/templates/allow_debug.html

@@ -3,29 +3,11 @@
 {% block title %}Debug allow rules{% endblock %}
 
 {% block extra_head %}
+{% include "_permission_ui_styles.html" %}
 <style>
-textarea {
-    height: 10em;
-    width: 95%;
-    box-sizing: border-box;
-    padding: 0.5em;
-    border: 2px dotted black;
-}
-.two-col {
-    display: inline-block;
-    width: 48%;
-}
-.two-col label {
-    width: 48%;
-}
 p.message-warning {
     white-space: pre-wrap;
 }
-@media only screen and (max-width: 576px) {
-    .two-col {
-        width: 100%;
-    }
-}
 </style>
 {% endblock %}
 
@@ -38,24 +20,28 @@ p.message-warning {
 
 <p>Use this tool to try out different actor and allow combinations. See <a href="https://docs.datasette.io/en/stable/authentication.html#defining-permissions-with-allow-blocks">Defining permissions with "allow" blocks</a> for documentation.</p>
 
-<form class="core" action="{{ urls.path('-/allow-debug') }}" method="get" style="margin-bottom: 1em">
-    <div class="two-col">
-        <p><label>Allow block</label></p>
-        <textarea name="allow">{{ allow_input }}</textarea>
-    </div>
-    <div class="two-col">
-        <p><label>Actor</label></p>
-        <textarea name="actor">{{ actor_input }}</textarea>
-    </div>
-    <div style="margin-top: 1em;">
-        <input type="submit" value="Apply allow block to actor">
-    </div>
-</form>
+<div class="permission-form">
+    <form class="core" action="{{ urls.path('-/allow-debug') }}" method="get">
+        <div class="permission-form-grid">
+            <div class="form-section">
+                <label for="allow-block">Allow block</label>
+                <textarea class="permission-textarea" id="allow-block" name="allow">{{ allow_input }}</textarea>
+            </div>
+            <div class="form-section">
+                <label for="allow-actor">Actor</label>
+                <textarea class="permission-textarea" id="allow-actor" name="actor">{{ actor_input }}</textarea>
+            </div>
+        </div>
+        <div class="form-actions">
+            <button type="submit" class="submit-btn">Apply allow block to actor</button>
+        </div>
+    </form>
 
-{% if error %}<p class="message-warning">{{ error }}</p>{% endif %}
+    {% if error %}<p class="message-warning permission-form-result">{{ error }}</p>{% endif %}
 
-{% if result == "True" %}<p class="message-info">Result: allow</p>{% endif %}
+    {% if result == "True" %}<p class="message-info permission-form-result">Result: allow</p>{% endif %}
 
-{% if result == "False" %}<p class="message-error">Result: deny</p>{% endif %}
+    {% if result == "False" %}<p class="message-error permission-form-result">Result: deny</p>{% endif %}
+</div>
 
 {% endblock %}

datasette/templates/api_explorer.html

@@ -3,7 +3,7 @@
 {% block title %}API Explorer{% endblock %}
 
 {% block extra_head %}
-<script src="{{ base_url }}-/static/json-format-highlight-1.0.1.js"></script>
+<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% endblock %}
 
 {% block content %}

datasette/templates/base.html

@@ -2,13 +2,13 @@
 <html lang="en">
 <head>
     <title>{% block title %}{% endblock %}</title>
-    <link rel="stylesheet" href="{{ urls.static('app.css') }}?{{ app_css_hash }}">
+    <link rel="stylesheet" href="{{ static('app.css') }}">
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
 {% for url in extra_css_urls %}
     <link rel="stylesheet" href="{{ url.url }}"{% if url.get("sri") %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}>
 {% endfor %}
 <script>window.datasetteVersion = '{{ datasette_version }}';</script>
-<script src="{{ urls.static('datasette-manager.js') }}" defer></script>
+<script src="{{ static('datasette-manager.js') }}" defer></script>
 {% for url in extra_js_urls %}
     <script {% if url.module %}type="module" {% endif %}src="{{ url.url }}"{% if url.get("sri") %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}></script>
 {% endfor %}
@@ -70,7 +70,7 @@
 {% endfor %}
 
 {% if select_templates %}<!-- Templates considered: {{ select_templates|join(", ") }} -->{% endif %}
-<script src="{{ urls.static('navigation-search.js') }}" defer></script>
+<script src="{{ static('navigation-search.js') }}" defer></script>
 <navigation-search url="{{ urls.path("/-/jump") }}"></navigation-search>
 </body>
 </html>

datasette/templates/database.html

@@ -6,6 +6,10 @@
 {{- super() -}}
 {% include "_codemirror.html" %}
 {% include "_sql_parameter_styles.html" %}
+{% if database_page_data.createTable %}
+<script>window._datasetteDatabaseData = {{ database_page_data|tojson }};</script>
+<script src="{{ static('edit-tools.js') }}" defer></script>
+{% endif %}
 {% endblock %}
 
 {% block body_class %}db db-{{ database|to_css_class }}{% endblock %}
@@ -72,7 +76,7 @@
 <div class="db-table">
     <h3><a href="{{ urls.table(database, table.name) }}">{{ table.name }}</a>{% if table.private %} 🔒{% endif %}{% if table.hidden %}<em> (hidden)</em>{% endif %}</h3>
     <p><em>{% for column in table.columns %}{{ column }}{% if not loop.last %}, {% endif %}{% endfor %}</em></p>
-    <p>{% if table.count is none %}Many rows{% elif table.count == count_limit + 1 %}&gt;{{ "{:,}".format(count_limit) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}</p>
+    <p>{% if table.count is none %}Many rows{% elif table.count_truncated %}&gt;{{ "{:,}".format(table.count - 1) }} rows{% else %}{{ "{:,}".format(table.count) }} row{% if table.count == 1 %}{% else %}s{% endif %}{% endif %}</p>
 </div>
 {% endif %}
 {% endfor %}

datasette/templates/debug_actions.html

@@ -9,7 +9,7 @@
 {% include "_permissions_debug_tabs.html" %}
 
 <p style="margin-bottom: 2em;">
-  This Datasette instance has registered {{ data|length }} action{{ data|length != 1 and "s" or "" }}.
+  This Datasette instance has registered {{ data.actions|length }} action{{ data.actions|length != 1 and "s" or "" }}.
   Actions are used by the permission system to control access to different features.
 </p>
 
@@ -26,7 +26,7 @@
     </tr>
   </thead>
   <tbody>
-    {% for action in data %}
+    {% for action in data.actions %}
     <tr>
       <td><strong>{{ action.name }}</strong></td>
       <td>{% if action.abbr %}<code>{{ action.abbr }}</code>{% endif %}</td>

datasette/templates/debug_allowed.html

@@ -3,7 +3,7 @@
 {% block title %}Allowed Resources{% endblock %}
 
 {% block extra_head %}
-<script src="{{ base_url }}-/static/json-format-highlight-1.0.1.js"></script>
+<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -49,7 +49,7 @@
 
         <div class="form-section">
             <label for="page_size">Page size:</label>
-            <input type="number" id="page_size" name="page_size" value="50" min="1" max="200" style="max-width: 100px;">
+            <input type="number" id="page_size" name="_size" value="50" min="1" max="200">
             <small>Number of results per page (max 200)</small>
         </div>
 
@@ -88,7 +88,7 @@ const hasDebugPermission = {{ 'true' if has_debug_permission else 'false' }};
 (function() {
     const params = populateFormFromURL();
     const action = params.get('action');
-    const page = params.get('page');
+    const page = params.get('_page');
     if (action) {
         fetchResults(page ? parseInt(page) : 1);
     }
@@ -102,14 +102,14 @@ async function fetchResults(page = 1) {
     const params = new URLSearchParams();
 
     for (const [key, value] of formData.entries()) {
-        if (value && key !== 'page_size') {
+        if (value && key !== '_size' && key !== '_page') {
             params.append(key, value);
         }
     }
 
     const pageSize = document.getElementById('page_size').value || '50';
-    params.append('page', page.toString());
-    params.append('page_size', pageSize);
+    params.append('_page', page.toString());
+    params.append('_size', pageSize);
 
     try {
         const response = await fetch('{{ urls.path("-/allowed.json") }}?' + params.toString(), {

datasette/templates/debug_autocomplete.html

@@ -0,0 +1,78 @@
+{% extends "base.html" %}
+
+{% block title %}Debug autocomplete{% endblock %}
+
+{% block extra_head %}
+{{ super() }}
+<script src="{{ static('autocomplete.js') }}" defer></script>
+{% endblock %}
+
+{% block content %}
+<h1>Debug autocomplete</h1>
+
+<form class="core debug-autocomplete-form" action="{{ urls.path('-/debug/autocomplete') }}" method="get">
+    <p>
+        <label for="debug-autocomplete-database">Database</label>
+        <input id="debug-autocomplete-database" type="text" name="database" value="{{ database_name or "" }}">
+    </p>
+    <p>
+        <label for="debug-autocomplete-table">Table</label>
+        <input id="debug-autocomplete-table" type="text" name="table" value="{{ table_name or "" }}">
+    </p>
+    <p><input type="submit" value="Open autocomplete"></p>
+</form>
+
+{% if error %}
+    <p class="message-error">{{ error }}</p>
+{% elif autocomplete_url %}
+    <h2>{{ database_name }} / {{ table_name }}</h2>
+    {% if label_column %}
+        <p>Label column: <code>{{ label_column }}</code></p>
+    {% else %}
+        <p>No label column detected. Results will use primary key values.</p>
+    {% endif %}
+    <div class="debug-autocomplete-demo">
+        <label for="debug-autocomplete-input">Search rows</label>
+        <datasette-autocomplete src="{{ autocomplete_url }}">
+            <input id="debug-autocomplete-input" type="text">
+        </datasette-autocomplete>
+    </div>
+    <h3>Selected row</h3>
+    <pre class="debug-autocomplete-selected" aria-live="polite">No row selected.</pre>
+    <script>
+    document.addEventListener("datasette-autocomplete-select", function (event) {
+        var output = document.querySelector(".debug-autocomplete-selected");
+        if (output) {
+            output.textContent = JSON.stringify(event.detail.row, null, 2);
+        }
+    });
+    </script>
+{% else %}
+    <h2>Suggested tables</h2>
+    {% if suggestions %}
+        <p>Showing up to five tables with a detected label column.</p>
+        <table class="rows-and-columns">
+            <thead>
+                <tr>
+                    <th>Database</th>
+                    <th>Table</th>
+                    <th>Label column</th>
+                </tr>
+            </thead>
+            <tbody>
+                {% for suggestion in suggestions %}
+                    <tr>
+                        <td>{{ suggestion.database }}</td>
+                        <td><a href="{{ suggestion.url }}">{{ suggestion.table }}</a></td>
+                        <td><code>{{ suggestion.label_column }}</code></td>
+                    </tr>
+                {% endfor %}
+            </tbody>
+        </table>
+    {% else %}
+        <p>No tables with detected label columns found.</p>
+    {% endif %}
+    <p>Scanned {{ scanned }} table{% if scanned != 1 %}s{% endif %}{% if reached_scan_limit %}; stopped at the 100 table scan limit{% endif %}.</p>
+{% endif %}
+
+{% endblock %}

datasette/templates/debug_check.html

@@ -1,9 +1,9 @@
 {% extends "base.html" %}
 
-{% block title %}Permission Check{% endblock %}
+{% block title %}Explain a permission decision{% endblock %}
 
 {% block extra_head %}
-<script src="{{ base_url }}-/static/json-format-highlight-1.0.1.js"></script>
+<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 <style>
@@ -13,29 +13,35 @@
     border-radius: 5px;
 }
 #output.allowed {
-    background-color: #e8f5e9;
+    background-color: #f3fbf4;
     border: 2px solid #4caf50;
 }
 #output.denied {
-    background-color: #ffebee;
+    background-color: #fff7f7;
     border: 2px solid #f44336;
 }
 #output h2 {
     margin-top: 0;
 }
-#output .result-badge {
+#output h3 {
+    margin-bottom: 0.5em;
+}
+#output .result-badge,
+.effect-badge,
+.rule-status {
     display: inline-block;
-    padding: 0.3em 0.8em;
+    padding: 0.2em 0.5em;
     border-radius: 3px;
     font-weight: bold;
-    font-size: 1.1em;
 }
-#output .allowed-badge {
-    background-color: #4caf50;
+#output .allowed-badge,
+.effect-allow {
+    background-color: #2e7d32;
     color: white;
 }
-#output .denied-badge {
-    background-color: #f44336;
+#output .denied-badge,
+.effect-deny {
+    background-color: #c62828;
     color: white;
 }
 .details-section {
@@ -48,70 +54,130 @@
 .details-section dd {
     margin-left: 1em;
 }
+.explanation-section {
+    background: rgba(255, 255, 255, 0.75);
+    border: 1px solid #ddd;
+    border-radius: 4px;
+    margin-top: 1em;
+    padding: 0 1em 1em;
+}
+.rules-table {
+    border-collapse: collapse;
+    width: 100%;
+}
+.rules-table th,
+.rules-table td {
+    border-bottom: 1px solid #ddd;
+    padding: 0.5em;
+    text-align: left;
+    vertical-align: top;
+}
+.rule-status {
+    background: #e8f5e9;
+    color: #1b5e20;
+}
+.rule-ignored {
+    background: #eee;
+    color: #555;
+    font-weight: normal;
+}
+.requirement-allowed {
+    color: #1b5e20;
+}
+.requirement-denied {
+    color: #b71c1c;
+}
+@media only screen and (max-width: 576px) {
+    .rules-table,
+    .rules-table tbody,
+    .rules-table tr,
+    .rules-table td {
+        display: block;
+    }
+    .rules-table thead {
+        display: none;
+    }
+    .rules-table td::before {
+        content: attr(data-label) ": ";
+        font-weight: bold;
+    }
+}
 </style>
 {% endblock %}
 
 {% block content %}
-<h1>Permission check</h1>
+<h1>Explain a permission decision</h1>
 
 {% set current_tab = "check" %}
 {% include "_permissions_debug_tabs.html" %}
 
-<p>Use this tool to test permission checks for the current actor. It queries the <code>/-/check.json</code> API endpoint.</p>
-
-{% if request.actor %}
-<p>Current actor: <strong>{{ request.actor.get("id", "anonymous") }}</strong></p>
-{% else %}
-<p>Current actor: <strong>anonymous (not logged in)</strong></p>
-{% endif %}
+<p>Test an actor, action and resource. The result explains which rules matched, which specificity level won, and whether actor restrictions or required actions changed the verdict.</p>
 
 <div class="permission-form">
-    <form id="check-form" method="get" action="{{ urls.path("-/check") }}">
+    <form id="check-form" method="get" action="{{ urls.path('-/check') }}">
         <div class="form-section">
-            <label for="action">Action (permission name):</label>
+            <label for="actor">Actor JSON:</label>
+            <textarea class="permission-textarea" id="actor" name="actor">{{ actor_json }}</textarea>
+            <small>Use <code>null</code> for an anonymous actor. This actor is simulated; it does not change who you are signed in as.</small>
+        </div>
+
+        <div class="form-section">
+            <label for="action">Action:</label>
             <select id="action" name="action" required>
                 <option value="">Select an action...</option>
-                {% for action_name in sorted_actions %}
-                <option value="{{ action_name }}">{{ action_name }}</option>
+                {% for action in actions %}
+                <option value="{{ action.name }}">{{ action.name }}{% if action.description %} — {{ action.description }}{% endif %}</option>
                 {% endfor %}
             </select>
-            <small>The permission action to check</small>
+            <small id="action-help">The operation to evaluate</small>
         </div>
 
-        <div class="form-section">
-            <label for="parent">Parent resource (optional):</label>
+        <div class="form-section" id="parent-section">
+            <label for="parent">Parent resource:</label>
             <input type="text" id="parent" name="parent" placeholder="e.g., database name">
-            <small>For database-level permissions, specify the database name</small>
+            <small>The database or other parent resource</small>
         </div>
 
-        <div class="form-section">
-            <label for="child">Child resource (optional):</label>
-            <input type="text" id="child" name="child" placeholder="e.g., table name">
-            <small>For table-level permissions, specify the table name (requires parent)</small>
+        <div class="form-section" id="child-section">
+            <label for="child">Child resource:</label>
+            <input type="text" id="child" name="child" placeholder="e.g., table or query name">
+            <small>The table, query or other child resource</small>
         </div>
 
         <div class="form-actions">
-            <button type="submit" class="submit-btn" id="submit-btn">Check Permission</button>
+            <button type="submit" class="submit-btn" id="submit-btn">Explain decision</button>
         </div>
     </form>
 </div>
 
 <div id="output" style="display: none;">
     <h2>Result: <span class="result-badge" id="result-badge"></span></h2>
+    <p id="result-summary"></p>
 
     <dl class="details-section">
+        <dt>Actor:</dt>
+        <dd><code id="result-actor"></code></dd>
         <dt>Action:</dt>
-        <dd id="result-action"></dd>
-
-        <dt>Resource Path:</dt>
-        <dd id="result-resource"></dd>
-
-        <dt>Actor ID:</dt>
-        <dd id="result-actor"></dd>
-
-        <div id="additional-details"></div>
+        <dd><code id="result-action"></code></dd>
+        <dt>Resource:</dt>
+        <dd><code id="result-resource"></code></dd>
     </dl>
 
+    <section class="explanation-section">
+        <h3>Matching rules</h3>
+        <div id="matching-rules"></div>
+    </section>
+
+    <section class="explanation-section" id="restrictions-section">
+        <h3>Actor restrictions</h3>
+        <div id="restriction-results"></div>
+    </section>
+
+    <section class="explanation-section" id="requirements-section">
+        <h3>Required actions</h3>
+        <div id="requirement-results"></div>
+    </section>
+
     <details style="margin-top: 1em;">
         <summary style="cursor: pointer; font-weight: bold;">Raw JSON response</summary>
         <pre id="raw-json" style="margin-top: 1em; padding: 1em; background-color: #f5f5f5; border: 1px solid #ddd; border-radius: 3px; overflow-x: auto;"></pre>
@@ -119,152 +185,134 @@
 </div>
 
 <script>
+const actions = Object.fromEntries({{ actions|tojson }}.map(action => [action.name, action]));
 const form = document.getElementById('check-form');
 const output = document.getElementById('output');
 const submitBtn = document.getElementById('submit-btn');
+const actionSelect = document.getElementById('action');
+
+function updateResourceFields() {
+    const action = actions[actionSelect.value];
+    document.getElementById('parent-section').style.display = action && action.takes_parent ? 'block' : 'none';
+    document.getElementById('child-section').style.display = action && action.takes_child ? 'block' : 'none';
+    let help = action && action.description ? action.description : 'The operation to evaluate';
+    if (action && action.also_requires) {
+        help += `; also requires ${action.also_requires}`;
+    }
+    document.getElementById('action-help').textContent = help;
+}
 
 async function performCheck() {
     submitBtn.disabled = true;
-    submitBtn.textContent = 'Checking...';
-
-    const formData = new FormData(form);
-    const params = new URLSearchParams();
-
-    for (const [key, value] of formData.entries()) {
-        if (value) {
-            params.append(key, value);
-        }
-    }
+    submitBtn.textContent = 'Explaining...';
+    const params = new URLSearchParams(new FormData(form));
 
     try {
         const response = await fetch('{{ urls.path("-/check.json") }}?' + params.toString(), {
-            method: 'GET',
-            headers: {
-                'Accept': 'application/json',
-            }
+            headers: {'Accept': 'application/json'}
         });
-
         const data = await response.json();
-
         if (response.ok) {
             displayResult(data);
         } else {
             displayError(data);
         }
     } catch (error) {
-        alert('Error: ' + error.message);
+        displayError({error: error.message});
     } finally {
         submitBtn.disabled = false;
-        submitBtn.textContent = 'Check Permission';
+        submitBtn.textContent = 'Explain decision';
     }
 }
 
-// Populate form on initial load
-(function() {
-    const params = populateFormFromURL();
-    const action = params.get('action');
-    if (action) {
-        performCheck();
-    }
-})();
-
 function displayResult(data) {
     output.style.display = 'block';
-
-    // Set badge and styling
     const resultBadge = document.getElementById('result-badge');
-    if (data.allowed) {
-        output.className = 'allowed';
-        resultBadge.className = 'result-badge allowed-badge';
-        resultBadge.textContent = 'ALLOWED ✓';
-    } else {
-        output.className = 'denied';
-        resultBadge.className = 'result-badge denied-badge';
-        resultBadge.textContent = 'DENIED ✗';
-    }
-
-    // Basic details
-    document.getElementById('result-action').textContent = data.action || 'N/A';
-    document.getElementById('result-resource').textContent = data.resource?.path || '/';
-    document.getElementById('result-actor').textContent = data.actor_id || 'anonymous';
-
-    // Additional details
-    const additionalDetails = document.getElementById('additional-details');
-    additionalDetails.innerHTML = '';
-
-    if (data.reason !== undefined) {
-        const dt = document.createElement('dt');
-        dt.textContent = 'Reason:';
-        const dd = document.createElement('dd');
-        dd.textContent = data.reason || 'N/A';
-        additionalDetails.appendChild(dt);
-        additionalDetails.appendChild(dd);
-    }
-
-    if (data.source_plugin !== undefined) {
-        const dt = document.createElement('dt');
-        dt.textContent = 'Source Plugin:';
-        const dd = document.createElement('dd');
-        dd.textContent = data.source_plugin || 'N/A';
-        additionalDetails.appendChild(dt);
-        additionalDetails.appendChild(dd);
-    }
-
-    if (data.used_default !== undefined) {
-        const dt = document.createElement('dt');
-        dt.textContent = 'Used Default:';
-        const dd = document.createElement('dd');
-        dd.textContent = data.used_default ? 'Yes' : 'No';
-        additionalDetails.appendChild(dt);
-        additionalDetails.appendChild(dd);
-    }
-
-    if (data.depth !== undefined) {
-        const dt = document.createElement('dt');
-        dt.textContent = 'Depth:';
-        const dd = document.createElement('dd');
-        dd.textContent = data.depth;
-        additionalDetails.appendChild(dt);
-        additionalDetails.appendChild(dd);
-    }
-
-    // Raw JSON
+    output.className = data.allowed ? 'allowed' : 'denied';
+    resultBadge.className = `result-badge ${data.allowed ? 'allowed-badge' : 'denied-badge'}`;
+    resultBadge.textContent = data.allowed ? 'ALLOWED ✓' : 'DENIED ✗';
+    document.getElementById('result-summary').textContent = data.explanation.summary;
+    document.getElementById('result-actor').textContent = data.actor === null ? 'anonymous' : JSON.stringify(data.actor);
+    document.getElementById('result-action').textContent = data.action;
+    document.getElementById('result-resource').textContent = data.resource.path;
+    displayRules(data.explanation);
+    displayRestrictions(data.explanation.restrictions);
+    displayRequirements(data.explanation.required_actions);
     document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+}
 
-    // Scroll to output
-    output.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
+function displayRules(explanation) {
+    const container = document.getElementById('matching-rules');
+    if (!explanation.matched_rules.length) {
+        container.innerHTML = '<p>No rules matched. Datasette denies access when there is no matching rule.</p>';
+        return;
+    }
+    let html = '<table class="rules-table"><thead><tr><th>Effect</th><th>Scope</th><th>Source</th><th>Reason</th><th>Role in decision</th></tr></thead><tbody>';
+    for (const rule of explanation.matched_rules) {
+        const status = rule.decisive
+            ? '<span class="rule-status">Decisive</span>'
+            : `<span class="rule-status rule-ignored">${escapeHtml(rule.ignored_because)}</span>`;
+        html += '<tr>';
+        html += `<td data-label="Effect"><span class="effect-badge effect-${rule.effect}">${rule.effect.toUpperCase()}</span></td>`;
+        html += `<td data-label="Scope">${escapeHtml(rule.scope)}</td>`;
+        html += `<td data-label="Source"><code>${escapeHtml(rule.source || 'unknown')}</code></td>`;
+        html += `<td data-label="Reason">${escapeHtml(rule.reason || 'No reason supplied')}</td>`;
+        html += `<td data-label="Role in decision">${status}</td>`;
+        html += '</tr>';
+    }
+    container.innerHTML = html + '</tbody></table>';
+}
+
+function displayRestrictions(restrictions) {
+    const section = document.getElementById('restrictions-section');
+    const container = document.getElementById('restriction-results');
+    section.style.display = restrictions.length ? 'block' : 'none';
+    container.innerHTML = restrictions.map(restriction => {
+        const className = restriction.allowed ? 'requirement-allowed' : 'requirement-denied';
+        const verdict = restriction.allowed ? 'INCLUDED ✓' : 'EXCLUDED ✗';
+        return `<p class="${className}"><strong>${verdict}</strong> by <code>${escapeHtml(restriction.source || 'unknown')}</code>: ${escapeHtml(restriction.reason)}</p>`;
+    }).join('');
+}
+
+function displayRequirements(requirements) {
+    const section = document.getElementById('requirements-section');
+    const container = document.getElementById('requirement-results');
+    section.style.display = requirements.length ? 'block' : 'none';
+    container.innerHTML = requirements.map(requirement => {
+        const className = requirement.allowed ? 'requirement-allowed' : 'requirement-denied';
+        const verdict = requirement.allowed ? 'ALLOWED ✓' : 'DENIED ✗';
+        return `<p class="${className}"><strong>${escapeHtml(requirement.action)}: ${verdict}</strong> — ${escapeHtml(requirement.summary)}</p>`;
+    }).join('');
 }
 
 function displayError(data) {
     output.style.display = 'block';
     output.className = 'denied';
-
     const resultBadge = document.getElementById('result-badge');
     resultBadge.className = 'result-badge denied-badge';
     resultBadge.textContent = 'ERROR';
-
-    document.getElementById('result-action').textContent = 'N/A';
-    document.getElementById('result-resource').textContent = 'N/A';
-    document.getElementById('result-actor').textContent = 'N/A';
-
-    const additionalDetails = document.getElementById('additional-details');
-    additionalDetails.innerHTML = '<dt>Error:</dt><dd>' + (data.error || 'Unknown error') + '</dd>';
-
+    document.getElementById('result-summary').textContent = data.error || 'Unknown error';
+    document.getElementById('result-actor').textContent = '—';
+    document.getElementById('result-action').textContent = '—';
+    document.getElementById('result-resource').textContent = '—';
+    document.getElementById('matching-rules').innerHTML = '';
+    document.getElementById('restrictions-section').style.display = 'none';
+    document.getElementById('requirements-section').style.display = 'none';
     document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
-
-    output.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
 }
 
-// Disable child input if parent is empty
-const parentInput = document.getElementById('parent');
-const childInput = document.getElementById('child');
-
-childInput.addEventListener('focus', () => {
-    if (!parentInput.value) {
-        alert('Please specify a parent resource first before adding a child resource.');
-        parentInput.focus();
-    }
+form.addEventListener('submit', event => {
+    event.preventDefault();
+    performCheck();
 });
-</script>
+actionSelect.addEventListener('change', updateResourceFields);
 
+(function initializeFromUrl() {
+    const params = populateFormFromURL();
+    updateResourceFields();
+    if (params.get('action')) {
+        performCheck();
+    }
+})();
+</script>
 {% endblock %}

datasette/templates/debug_permissions_playground.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 
-{% block title %}Debug permissions{% endblock %}
+{% block title %}Permission activity{% endblock %}
 
 {% block extra_head %}
 {% include "_permission_ui_styles.html" %}
@@ -20,60 +20,45 @@
 .check-action, .check-when, .check-result {
     font-size: 1.3em;
 }
-textarea {
-    height: 10em;
-    width: 95%;
-    box-sizing: border-box;
-    padding: 0.5em;
-    border: 2px dotted black;
-}
-.two-col {
-    display: inline-block;
-    width: 48%;
-}
-.two-col label {
-    width: 48%;
-}
-@media only screen and (max-width: 576px) {
-    .two-col {
-        width: 100%;
-    }
-}
 </style>
 {% endblock %}
 
 {% block content %}
-<h1>Permission playground</h1>
+<h1>Permission activity</h1>
 
 {% set current_tab = "permissions" %}
 {% include "_permissions_debug_tabs.html" %}
 
-<p>This tool lets you simulate an actor and a permission check for that actor.</p>
+<h2>Raw simulator</h2>
+
+<p>This form runs a hypothetical permission check and returns its raw explanation JSON. Use the <a href="{{ urls.path('-/check') }}">Explain tool</a> for a visual explanation of the same decision.</p>
 
 <div class="permission-form">
     <form action="{{ urls.path('-/permissions') }}" id="debug-post" method="post">
-        <div class="two-col">
-            <div class="form-section">
-                <label>Actor</label>
-                <textarea name="actor">{% if actor_input %}{{ actor_input }}{% else %}{"id": "root"}{% endif %}</textarea>
+        <div class="permission-form-grid">
+            <div>
+                <div class="form-section">
+                    <label for="activity-actor">Actor</label>
+                    <textarea class="permission-textarea" id="activity-actor" name="actor">{% if actor_input %}{{ actor_input }}{% else %}{"id": "root"}{% endif %}</textarea>
+                </div>
             </div>
-        </div>
-        <div class="two-col" style="vertical-align: top">
-            <div class="form-section">
-                <label for="permission">Action</label>
-                <select name="permission" id="permission">
-                    {% for permission in permissions %}
-                        <option value="{{ permission.name }}">{{ permission.name }}</option>
-                    {% endfor %}
-                </select>
-            </div>
-            <div class="form-section">
-                <label for="resource_1">Parent</label>
-                <input type="text" id="resource_1" name="resource_1" placeholder="e.g., database name">
-            </div>
-            <div class="form-section">
-                <label for="resource_2">Child</label>
-                <input type="text" id="resource_2" name="resource_2" placeholder="e.g., table name">
+            <div>
+                <div class="form-section">
+                    <label for="permission">Action</label>
+                    <select name="permission" id="permission">
+                        {% for permission in permissions %}
+                            <option value="{{ permission.name }}">{{ permission.name }}</option>
+                        {% endfor %}
+                    </select>
+                </div>
+                <div class="form-section">
+                    <label for="resource_1">Parent</label>
+                    <input type="text" id="resource_1" name="resource_1" placeholder="e.g., database name">
+                </div>
+                <div class="form-section">
+                    <label for="resource_2">Child</label>
+                    <input type="text" id="resource_2" name="resource_2" placeholder="e.g., table name">
+                </div>
             </div>
         </div>
         <div class="form-actions">
@@ -125,7 +110,7 @@ debugPost.addEventListener('submit', function(ev) {
 });
 </script>
 
-<h1>Recent permissions checks</h1>
+<h2>Recent permission checks</h2>
 
 <p>
     {% if filter != "all" %}<a href="?filter=all">All</a>{% else %}<strong>All</strong>{% endif %},

datasette/templates/debug_rules.html

@@ -3,7 +3,7 @@
 {% block title %}Permission Rules{% endblock %}
 
 {% block extra_head %}
-<script src="{{ base_url }}-/static/json-format-highlight-1.0.1.js"></script>
+<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -37,7 +37,7 @@
 
         <div class="form-section">
             <label for="page_size">Page size:</label>
-            <input type="number" id="page_size" name="page_size" value="50" min="1" max="200" style="max-width: 100px;">
+            <input type="number" id="page_size" name="_size" value="50" min="1" max="200">
             <small>Number of results per page (max 200)</small>
         </div>
 
@@ -75,7 +75,7 @@ const submitBtn = document.getElementById('submit-btn');
 (function() {
     const params = populateFormFromURL();
     const action = params.get('action');
-    const page = params.get('page');
+    const page = params.get('_page');
     if (action) {
         fetchResults(page ? parseInt(page) : 1);
     }
@@ -89,14 +89,14 @@ async function fetchResults(page = 1) {
     const params = new URLSearchParams();
 
     for (const [key, value] of formData.entries()) {
-        if (value && key !== 'page_size') {
+        if (value && key !== '_size' && key !== '_page') {
             params.append(key, value);
         }
     }
 
     const pageSize = document.getElementById('page_size').value || '50';
-    params.append('page', page.toString());
-    params.append('page_size', pageSize);
+    params.append('_page', page.toString());
+    params.append('_size', pageSize);
 
     try {
         const response = await fetch('{{ urls.path("-/rules.json") }}?' + params.toString(), {

datasette/templates/execute_write.html

@@ -56,6 +56,11 @@ form.sql.core input[data-execute-write-submit]:disabled {
     cursor: not-allowed;
     opacity: 1;
 }
+.execute-write form.sql .sql-editor-min-lines .cm-content,
+.execute-write form.sql .sql-editor-min-lines .cm-gutter {
+    /* Four visible editor lines without adding blank lines to the SQL value. */
+    min-height: calc(5.6em + 8px);
+}
 .execute-write-disabled-reason {
     color: #4f5b6d;
     font-size: 0.85rem;
@@ -93,20 +98,25 @@ form.sql.core input[data-execute-write-submit]:disabled {
 {% endif %}
 
 <form class="sql core" action="{{ urls.database(database) }}/-/execute-write" method="post" data-analyze-url="{{ urls.database(database) }}/-/execute-write/analyze">
-    {% if write_template_tables %}
+    {% if write_create_table_template_sql or write_template_tables %}
         <div class="execute-write-template-menu">
             <details>
                 <summary>Start with a template</summary>
                 <p class="execute-write-template-controls">
-                    <label for="execute-write-template-table">Table</label>
-                    <select id="execute-write-template-table">
-                        {% for table_name, table in write_template_tables|dictsort %}
-                            <option value="{{ table_name }}"{% for operation, template_sql in table.templates|dictsort %} data-template-{{ operation }}-sql="{{ template_sql }}"{% endfor %}>{{ table_name }}</option>
+                    {% if write_create_table_template_sql %}
+                        <button type="button" data-sql-template="create" data-template-sql="{{ write_create_table_template_sql }}">Create table</button>
+                    {% endif %}
+                    {% if write_template_tables %}
+                        <label for="execute-write-template-table">{% if write_create_table_template_sql %}or table:{% else %}Table{% endif %}</label>
+                        <select id="execute-write-template-table">
+                            {% for table_name, table in write_template_tables|dictsort %}
+                                <option value="{{ table_name }}"{% for operation, template_sql in table.templates|dictsort %} data-template-{{ operation }}-sql="{{ template_sql }}"{% endfor %}>{{ table_name }}</option>
+                            {% endfor %}
+                        </select>
+                        {% for operation in write_template_operations %}
+                            <button type="button" data-sql-template="{{ operation.name }}">{{ operation.label }}</button>
                         {% endfor %}
-                    </select>
-                    {% for operation in write_template_operations %}
-                        <button type="button" data-sql-template="{{ operation.name }}">{{ operation.label }}</button>
-                    {% endfor %}
+                    {% endif %}
                 </p>
             </details>
         </div>
@@ -114,7 +124,7 @@ form.sql.core input[data-execute-write-submit]:disabled {
         <p class="message-warning execute-write-template-unavailable">There are no tables that you can currently edit.</p>
     {% endif %}
 
-    <p class="sql-editor"><textarea id="sql-editor" name="sql"{% if sql %} style="height: {{ sql.split("\n")|length + 2 }}em"{% endif %}>{{ sql }}</textarea></p>
+    <p class="sql-editor{% if not sql %} sql-editor-min-lines{% endif %}"><textarea id="sql-editor" name="sql"{% if sql %} style="height: {{ sql.split("\n")|length + 2 }}em"{% endif %}>{{ sql }}</textarea></p>
 
     {% set sql_parameters_section_id = "execute-write-parameters-section" %}
     {% set sql_parameters_allow_expand = true %}
@@ -159,19 +169,13 @@ form.sql.core input[data-execute-write-submit]:disabled {
     </p>
 </form>
 
-<script>
-const executeWriteSqlInput = document.querySelector("textarea#sql-editor");
-if (executeWriteSqlInput && !executeWriteSqlInput.value) {
-  executeWriteSqlInput.value = "\n\n\n";
-}
-</script>
-
 {% include "_codemirror_foot.html" %}
 {% include "_sql_parameter_scripts.html" %}
 {% include "_execute_write_analysis_scripts.html" %}
 
 <script>
 window.addEventListener("DOMContentLoaded", () => {
+  const executeWriteSqlInput = document.querySelector("textarea#sql-editor");
   const form = document.querySelector("form.sql.core");
   const analysisSection = document.querySelector("#execute-write-analysis-section");
   const submitButton = form
@@ -252,11 +256,12 @@ window.addEventListener("DOMContentLoaded", () => {
 });
 </script>
 
-{% if write_template_tables %}
+{% if write_create_table_template_sql or write_template_tables %}
 <script>
 window.addEventListener("DOMContentLoaded", () => {
   const tableSelect = document.querySelector("#execute-write-template-table");
   const templateButtons = document.querySelectorAll("[data-sql-template]");
+  const sqlInput = document.querySelector("textarea#sql-editor");
 
   function dataKey(operation) {
     return `template${operation.charAt(0).toUpperCase()}${operation.slice(1)}Sql`;
@@ -266,26 +271,59 @@ window.addEventListener("DOMContentLoaded", () => {
     return tableSelect ? tableSelect.options[tableSelect.selectedIndex] : null;
   }
 
-  function templateSql(operation) {
+  function templateSql(button) {
+    if (button.dataset.templateSql) {
+      return button.dataset.templateSql;
+    }
+    const operation = button.dataset.sqlTemplate;
     const option = selectedOption();
     return option ? option.dataset[dataKey(operation)] || "" : "";
   }
 
   function updateTemplateButtons() {
     templateButtons.forEach((button) => {
-      button.hidden = !templateSql(button.dataset.sqlTemplate);
+      button.hidden = !templateSql(button);
     });
   }
 
+  function updateSqlUrl(sql) {
+    if (!window.history || !window.history.replaceState) {
+      return;
+    }
+    const url = new URL(window.location.href);
+    url.searchParams.set("sql", sql);
+    window.history.replaceState(null, "", url.toString());
+  }
+
+  function setEditorSql(sql) {
+    if (window.editor) {
+      window.editor.dispatch({
+        changes: {
+          from: 0,
+          to: window.editor.state.doc.length,
+          insert: sql,
+        },
+        selection: { anchor: sql.length },
+      });
+      window.editor.focus();
+      if (sqlInput) {
+        sqlInput.value = sql;
+      }
+    } else if (sqlInput) {
+      sqlInput.value = sql;
+      sqlInput.dispatchEvent(new Event("input", { bubbles: true }));
+      sqlInput.focus();
+    }
+    updateSqlUrl(sql);
+  }
+
   templateButtons.forEach((button) => {
     button.addEventListener("click", () => {
-      const sql = templateSql(button.dataset.sqlTemplate);
+      const sql = templateSql(button);
       if (!sql) {
         return;
       }
-      const url = new URL(window.location.href);
-      url.searchParams.set("sql", sql);
-      window.location.href = url.toString();
+      setEditorSql(sql);
     });
   });
   if (tableSelect) {

datasette/templates/patterns.html

@@ -2,7 +2,7 @@
 <html lang="en">
 <head>
     <title>Datasette: Pattern Portfolio</title>
-    <link rel="stylesheet" href="{{ base_url }}-/static/app.css?{{ app_css_hash }}">
+    <link rel="stylesheet" href="{{ static('app.css') }}">
     <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
     <meta name="robots" content="noindex">
     <style></style>
@@ -202,9 +202,9 @@
     <h3>3 rows
         where characteristic_id = 2
     </h3>
-    <form class="filters" action="{{ base_url }}fixtures/roadside_attraction_characteristics" method="get">
+    <form class="core filters" action="{{ base_url }}fixtures/roadside_attraction_characteristics" method="get">
     <div class="search-row"><label for="_search">Search:</label><input id="_search" type="search" name="_search" value=""></div>
-        <div class="filter-row">
+        <div class="filter-row filter-controls-row">
             <div class="select-wrapper">
                 <select name="_filter_column_1">
                     <option value="">- remove filter -</option>
@@ -238,7 +238,7 @@
                 </select>
             </div><input type="text" name="_filter_value_1" class="filter-value" value="2">
         </div>
-    <div class="filter-row">
+    <div class="filter-row filter-controls-row">
         <div class="select-wrapper">
             <select name="_filter_column">
                 <option value="">- column -</option>
@@ -272,8 +272,8 @@
             </select>
         </div><input type="text" name="_filter_value" class="filter-value">
     </div>
-    <div class="filter-row">
-            <div class="select-wrapper small-screen-only">
+    <div class="filter-row filter-actions-row">
+            <div class="select-wrapper">
                 <select name="_sort" id="sort_by">
                     <option value="">Sort...</option>
                     <option value="rowid" selected>Sort by rowid</option>
@@ -281,8 +281,8 @@
                     <option value="characteristic_id">Sort by characteristic_id</option>
                 </select>
             </div>
-            <label class="sort_by_desc small-screen-only"><input type="checkbox" name="_sort_by_desc"> descending</label>
-        <input type="submit" value="Apply">
+            <label class="sort_by_desc"><input type="checkbox" name="_sort_by_desc"> descending</label>
+        <input type="submit" value="Apply filters">
     </div>
   </form>
 

datasette/templates/row.html

@@ -4,6 +4,13 @@
 
 {% block extra_head %}
 {{- super() -}}
+{% if row_mutation_ui %}
+<script>window._datasetteTableData = {{ table_page_data|tojson }};</script>
+{% if table_page_data.foreignKeys %}
+<script src="{{ static('autocomplete.js') }}" defer></script>
+{% endif %}
+<script src="{{ static('edit-tools.js') }}" defer></script>
+{% endif %}
 <style>
 @media only screen and (max-width: 576px) {
 {% for column in columns %}

datasette/templates/table.html

@@ -1,12 +1,17 @@
 {% extends "base.html" %}
 
-{% block title %}{{ database }}: {{ table }}: {% if count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
+{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
 
 {% block extra_head %}
 {{- super() -}}
-<script src="{{ urls.static('column-chooser.js') }}" defer></script>
-<script src="{{ urls.static('table.js') }}" defer></script>
-<script src="{{ urls.static('mobile-column-actions.js') }}" defer></script>
+<script>window._datasetteTableData = {{ table_page_data|tojson }};</script>
+<script src="{{ static('column-chooser.js') }}" defer></script>
+{% if table_page_data.foreignKeys %}
+<script src="{{ static('autocomplete.js') }}" defer></script>
+{% endif %}
+<script src="{{ static('edit-tools.js') }}" defer></script>
+<script src="{{ static('table.js') }}" defer></script>
+<script src="{{ static('mobile-column-actions.js') }}" defer></script>
 <script>DATASETTE_ALLOW_FACET = {{ datasette_allow_facet }};</script>
 <style>
 @media only screen and (max-width: 576px) {
@@ -43,19 +48,19 @@
 
 {% if count or human_description_en %}
     <h3>
-        {% if count == count_limit + 1 %}&gt;{{ "{:,}".format(count_limit) }} rows
+        {% if count_truncated %}&gt;{{ "{:,}".format(count - 1) }} rows
         {% if allow_execute_sql and query.sql %} <a class="count-sql" style="font-size: 0.8em;" href="{{ urls.database_query(database, count_sql) }}">count all</a>{% endif %}
         {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}
         {% if human_description_en %}{{ human_description_en }}{% endif %}
     </h3>
 {% endif %}
 
-<form class="core" class="filters" action="{{ urls.table(database, table) }}" method="get">
+<form class="core filters" action="{{ urls.table(database, table) }}" method="get">
     {% if supports_search %}
         <div class="search-row"><label for="_search">Search:</label><input id="_search" type="search" name="_search" value="{{ search }}"></div>
     {% endif %}
     {% for column, lookup, value in filters.selections() %}
-        <div class="filter-row">
+        <div class="filter-row filter-controls-row">
             <div class="select-wrapper">
                 <select name="_filter_column_{{ loop.index }}">
                     <option value="">- remove filter -</option>
@@ -72,7 +77,7 @@
             </div><input type="text" name="_filter_value_{{ loop.index }}" class="filter-value" value="{{ value }}">
         </div>
     {% endfor %}
-    <div class="filter-row">
+    <div class="filter-row filter-controls-row">
         <div class="select-wrapper">
             <select name="_filter_column">
                 <option value="">- column -</option>
@@ -88,9 +93,9 @@
             </select>
         </div><input type="text" name="_filter_value" class="filter-value">
     </div>
-    <div class="filter-row">
+    <div class="filter-row filter-actions-row">
         {% if is_sortable %}
-            <div class="select-wrapper small-screen-only">
+            <div class="select-wrapper">
                 <select name="_sort" id="sort_by">
                     <option value="">Sort...</option>
                     {% for column in display_columns %}
@@ -100,12 +105,12 @@
                     {% endfor %}
                 </select>
             </div>
-            <label class="sort_by_desc small-screen-only"><input type="checkbox" name="_sort_by_desc"{% if sort_desc %} checked{% endif %}> descending</label>
+            <label class="sort_by_desc"><input type="checkbox" name="_sort_by_desc" tabindex="0"{% if sort_desc %} checked{% endif %}> descending</label>
         {% endif %}
         {% for key, value in form_hidden_args %}
             <input type="hidden" name="{{ key }}" value="{{ value }}">
         {% endfor %}
-        <input type="submit" value="Apply">
+        <input type="submit" value="Apply filters" tabindex="0">
     </div>
 </form>
 
@@ -158,6 +163,19 @@ window._setColumnTypeData = {{ set_column_type_ui|tojson }};
 </script>
 {% endif %}
 
+{% if table_insert_ui %}
+<div class="table-row-toolbar">
+    <button type="button" class="core table-insert-row" data-table-action="insert-row">
+        <svg class="row-inline-action-icon" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+            <rect width="18" height="18" x="3" y="3" rx="2"></rect>
+            <path d="M8 12h8"></path>
+            <path d="M12 8v8"></path>
+        </svg>
+        <span>Insert row</span>
+    </button>
+</div>
+{% endif %}
+
 {% include custom_table_templates %}
 
 {% if next_url %}

datasette/tokens.py

@@ -18,6 +18,21 @@ if TYPE_CHECKING:
     from datasette.app import Datasette
 
 
+class TokenInvalid(Exception):
+    """
+    Raised by a TokenHandler when a token it recognizes is invalid -
+    for example a bad signature, malformed payload or expired token.
+
+    Datasette responds to this with an HTTP 401 error. Handlers should
+    return None instead for tokens they do not recognize at all, so that
+    other registered handlers get a chance to verify them.
+    """
+
+    def __init__(self, message="Invalid token"):
+        self.message = message
+        super().__init__(message)
+
+
 @dataclasses.dataclass
 class TokenRestrictions:
     """
@@ -108,8 +123,12 @@ class TokenHandler:
 
     async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]:
         """
-        Verify a token and return an actor dict, or None if this handler
-        does not recognize the token.
+        Verify a token and return an actor dict.
+
+        Return None if this handler does not recognize the token at all,
+        so other handlers can try it. Raise TokenInvalid if the token is
+        recognized but invalid (bad signature, malformed, expired) - the
+        request will fail with a 401 error.
         """
         raise NotImplementedError
 
@@ -147,29 +166,32 @@ class SignedTokenHandler(TokenHandler):
     async def verify_token(self, datasette: "Datasette", token: str) -> Optional[dict]:
         prefix = "dstok_"
 
-        if not datasette.setting("allow_signed_tokens"):
+        if not token.startswith(prefix):
+            # Not one of our tokens - leave it for other handlers
             return None
 
+        if not datasette.setting("allow_signed_tokens"):
+            raise TokenInvalid(
+                "Signed tokens are not enabled for this Datasette instance"
+            )
+
         max_signed_tokens_ttl = datasette.setting("max_signed_tokens_ttl")
 
-        if not token.startswith(prefix):
-            return None
-
         raw = token[len(prefix) :]
         try:
             decoded = datasette.unsign(raw, namespace="token")
         except itsdangerous.BadSignature:
-            return None
+            raise TokenInvalid("Invalid token signature")
 
         if "t" not in decoded:
-            return None
+            raise TokenInvalid("Invalid token: no timestamp")
         created = decoded["t"]
         if not isinstance(created, int):
-            return None
+            raise TokenInvalid("Invalid token: invalid timestamp")
 
         duration = decoded.get("d")
         if duration is not None and not isinstance(duration, int):
-            return None
+            raise TokenInvalid("Invalid token: invalid duration")
 
         if (duration is None and max_signed_tokens_ttl) or (
             duration is not None
@@ -180,7 +202,7 @@ class SignedTokenHandler(TokenHandler):
 
         if duration:
             if time.time() - created > duration:
-                return None
+                raise TokenInvalid("Token has expired")
 
         actor = {"id": decoded["a"], "token": "dstok"}
 

datasette/tracer.py

@@ -27,8 +27,10 @@ def get_task_id():
 @contextmanager
 def trace_child_tasks():
     token = trace_task_id.set(get_task_id())
-    yield
-    trace_task_id.reset(token)
+    try:
+        yield
+    finally:
+        trace_task_id.reset(token)
 
 
 @contextmanager

datasette/utils/__init__.py

@@ -1,4 +1,5 @@
 import asyncio
+import binascii
 from contextlib import contextmanager
 import aiofiles
 import click
@@ -224,24 +225,71 @@ def compound_keys_after_sql(pks, start_index=0):
     return "({})".format("\n  or\n".join(or_clauses))
 
 
+@documented
 class CustomJSONEncoder(json.JSONEncoder):
+    """
+    The CustomJSONEncoder class handles serialization for objects commonly used by Datasette,
+    including SQLite cursors and binary blobs. Datasette uses it internally to serve .json endpoints,
+    and plugins that return JSON can use it to match Datasette's own handling.
+
+    Built-in types (text, numbers, lists, etc) are encoded the same as Python's built-in ``json`` module.
+
+    - ``sqlite3.Row`` becomes a tuple
+    - ``sqlite3.Cursor`` becomes a list
+
+    Binary blobs are encoded as an object, with the actual data base64-encoded,
+    like so: ::
+
+        {
+            "$base64": True,
+            "encoded": ...,
+        }
+
+    Example: https://latest.datasette.io/fixtures/binary_data.json
+    """
+
     def default(self, obj):
         if isinstance(obj, sqlite3.Row):
             return tuple(obj)
         if isinstance(obj, sqlite3.Cursor):
             return list(obj)
         if isinstance(obj, bytes):
-            # Does it encode to utf8?
-            try:
-                return obj.decode("utf8")
-            except UnicodeDecodeError:
-                return {
-                    "$base64": True,
-                    "encoded": base64.b64encode(obj).decode("latin1"),
-                }
+            return {
+                "$base64": True,
+                "encoded": base64.b64encode(obj).decode("latin1"),
+            }
         return json.JSONEncoder.default(self, obj)
 
 
+class WriteJsonValueError(ValueError):
+    pass
+
+
+def decode_write_json_cell(value):
+    if not isinstance(value, dict):
+        return value
+    keys = set(value.keys())
+    if keys == {"$raw"}:
+        return value["$raw"]
+    if keys == {"$base64", "encoded"} and value.get("$base64") is True:
+        encoded = value["encoded"]
+        if not isinstance(encoded, str):
+            raise WriteJsonValueError("$base64 encoded value must be a string")
+        try:
+            return base64.b64decode(encoded, validate=True)
+        except binascii.Error as ex:
+            raise WriteJsonValueError("Invalid $base64 encoded value") from ex
+    return value
+
+
+def decode_write_json_row(row):
+    return {key: decode_write_json_cell(value) for key, value in row.items()}
+
+
+def decode_write_json_rows(rows):
+    return [decode_write_json_row(row) for row in rows]
+
+
 @contextmanager
 def sqlite_timelimit(conn, ms):
     deadline = time.perf_counter() + (ms / 1000)
@@ -410,12 +458,7 @@ def escape_css_string(s):
 def escape_sqlite(s):
     if _boring_keyword_re.match(s) and (s.lower() not in reserved_words):
         return s
-    elif "]" in s:
-        # SQLite does not support escaping ] inside [bracket] quoting, so fall
-        # back to double-quote quoting (doubling any embedded ") - #2677
-        return '"{}"'.format(s.replace('"', '""'))
-    else:
-        return f"[{s}]"
+    return '"{}"'.format(s.replace('"', '""'))
 
 
 def make_dockerfile(
@@ -593,7 +636,7 @@ def detect_primary_keys(conn, table):
 
 
 def get_outbound_foreign_keys(conn, table):
-    infos = conn.execute(f"PRAGMA foreign_key_list([{table}])").fetchall()
+    infos = conn.execute(f"PRAGMA foreign_key_list({escape_sqlite(table)})").fetchall()
     fks = []
     for info in infos:
         if info is not None:
@@ -1245,10 +1288,20 @@ class StartupError(Exception):
     pass
 
 
-_single_line_comment_re = re.compile(r"--.*")
-_multi_line_comment_re = re.compile(r"/\*.*?\*/", re.DOTALL)
-_single_quote_re = re.compile(r"'(?:''|[^'])*'")
-_double_quote_re = re.compile(r'"(?:\"\"|[^"])*"')
+# Comments and string literals, matched in a single pass so that whichever
+# construct starts first "wins" - this ensures a comment marker inside a string
+# literal (or a quote inside a comment) does not confuse the parameter scan.
+_comments_and_strings_re = re.compile(
+    r"""
+    --[^\n]*            # single line comment
+    | /\*.*?(?:\*/|\Z)  # multi line comment, possibly to end-of-input
+    | '(?:''|[^'])*'    # single quoted string ('' escapes a quote)
+    | "(?:""|[^"])*"    # double quoted identifier ("" escapes a quote)
+    | \[(?:[^\]])*\]    # square-bracket quoted identifier
+    | `(?:``|[^`])*`    # backtick quoted identifier
+    """,
+    re.DOTALL | re.VERBOSE,
+)
 _named_param_re = re.compile(r":(\w+)")
 
 
@@ -1259,10 +1312,9 @@ def named_parameters(sql: str) -> List[str]:
 
     e.g. for ``select * from foo where id=:id`` this would return ``["id"]``
     """
-    sql = _single_line_comment_re.sub("", sql)
-    sql = _multi_line_comment_re.sub("", sql)
-    sql = _single_quote_re.sub("", sql)
-    sql = _double_quote_re.sub("", sql)
+    # Strip comments and string literals first so that any ":name" sequences
+    # inside them are not mistaken for named parameters
+    sql = _comments_and_strings_re.sub("", sql)
     # Extract parameters from what is left
     return _named_param_re.findall(sql)
 
@@ -1275,6 +1327,54 @@ async def derive_named_parameters(db: "Database", sql: str) -> List[str]:
     return named_parameters(sql)
 
 
+def parse_size_limit(value, default, maximum, name="_size"):
+    """
+    Parse a page-size parameter using the same semantics as the table
+    view's ?_size=: blank means default, "max" means maximum, integers
+    must be 0 or greater and no larger than maximum. Raises ValueError
+    with a message suitable for a 400 response.
+    """
+    if value in (None, ""):
+        return default
+    if value == "max":
+        return maximum
+    try:
+        size = int(value)
+        if size < 0:
+            raise ValueError
+    except ValueError:
+        raise ValueError("{} must be a positive integer".format(name))
+    if size > maximum:
+        raise ValueError("{} must be <= {}".format(name, maximum))
+    return size
+
+
+UNSTABLE_API_MESSAGE = (
+    "This API is not part of Datasette's stable interface and may change at any time"
+)
+
+
+def error_body(messages, status):
+    """
+    The canonical JSON error body used by every Datasette JSON error response:
+
+        {"ok": False, "error": "...", "errors": ["...", ...], "status": 400}
+
+    "error" is all of the messages joined with "; ", "errors" is the full
+    list, "status" matches the HTTP status code. Callers may add extra
+    context keys to the returned dictionary but must not remove these four.
+    """
+    if isinstance(messages, str):
+        messages = [messages]
+    messages = [str(message) for message in messages]
+    return {
+        "ok": False,
+        "error": "; ".join(messages),
+        "errors": messages,
+        "status": status,
+    }
+
+
 def add_cors_headers(headers):
     headers["Access-Control-Allow-Origin"] = "*"
     headers["Access-Control-Allow-Headers"] = "Authorization, Content-Type"
@@ -1548,6 +1648,17 @@ def md5_not_usedforsecurity(s):
 _etag_cache = {}
 
 
+def sha256_file(filepath, chunk_size=4096):
+    hasher = hashlib.sha256()
+    with open(filepath, "rb") as fp:
+        while True:
+            chunk = fp.read(chunk_size)
+            if not chunk:
+                break
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
 async def calculate_etag(filepath, chunk_size=4096):
     if filepath in _etag_cache:
         return _etag_cache[filepath]

datasette/utils/actions_sql.py

@@ -21,6 +21,8 @@ The core pattern is:
 - Across levels, child beats parent beats global
 """
 
+import asyncio
+import re
 from typing import TYPE_CHECKING
 
 from datasette.utils.permissions import gather_permission_sql_from_hooks
@@ -250,88 +252,62 @@ async def _build_single_action_sql(
                 ]
             )
 
-    # Continue with the cascading logic
-    query_parts.extend(
-        [
-            "child_lvl AS (",
-            "  SELECT b.parent, b.child,",
-            "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-            "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,",
-            "         json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,",
-            "         json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons",
-            "  FROM base b",
-            "  LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child = b.child",
-            "  GROUP BY b.parent, b.child",
-            "),",
-            "parent_lvl AS (",
-            "  SELECT b.parent, b.child,",
-            "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-            "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,",
-            "         json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,",
-            "         json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons",
-            "  FROM base b",
-            "  LEFT JOIN all_rules ar ON ar.parent = b.parent AND ar.child IS NULL",
-            "  GROUP BY b.parent, b.child",
-            "),",
-            "global_lvl AS (",
-            "  SELECT b.parent, b.child,",
-            "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-            "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow,",
-            "         json_group_array(CASE WHEN ar.allow = 0 THEN ar.source_plugin || ': ' || ar.reason END) AS deny_reasons,",
-            "         json_group_array(CASE WHEN ar.allow = 1 THEN ar.source_plugin || ': ' || ar.reason END) AS allow_reasons",
-            "  FROM base b",
-            "  LEFT JOIN all_rules ar ON ar.parent IS NULL AND ar.child IS NULL",
-            "  GROUP BY b.parent, b.child",
-            "),",
+    # Continue with the cascading logic.
+    # Aggregate the RULES by cascade level (small), rather than grouping
+    # base x rules (which scales with the number of resources).
+    def _agg(select_key, where, group_by):
+        parts = [
+            f"  SELECT {select_key}",
+            "         MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
+            "         MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow,",
+            "         json_group_array(CASE WHEN allow = 0 THEN source_plugin || ': ' || reason END) AS deny_reasons,",
+            "         json_group_array(CASE WHEN allow = 1 THEN source_plugin || ': ' || reason END) AS allow_reasons",
+            f"  FROM all_rules WHERE {where}",
         ]
+        if group_by:
+            parts.append(f"  GROUP BY {group_by}")
+        return parts
+
+    query_parts.extend(
+        ["child_agg AS ("]
+        + _agg(
+            "parent, child,",
+            "parent IS NOT NULL AND child IS NOT NULL",
+            "parent, child",
+        )
+        + ["),", "parent_agg AS ("]
+        + _agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent")
+        + ["),", "global_agg AS ("]
+        + _agg("", "parent IS NULL AND child IS NULL", None)
+        + ["),"]
     )
 
     # Add anonymous decision logic if needed
     if include_is_private:
-        query_parts.extend(
-            [
-                "anon_child_lvl AS (",
-                "  SELECT b.parent, b.child,",
-                "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-                "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow",
-                "  FROM base b",
-                "  LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child = b.child",
-                "  GROUP BY b.parent, b.child",
-                "),",
-                "anon_parent_lvl AS (",
-                "  SELECT b.parent, b.child,",
-                "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-                "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow",
-                "  FROM base b",
-                "  LEFT JOIN anon_rules ar ON ar.parent = b.parent AND ar.child IS NULL",
-                "  GROUP BY b.parent, b.child",
-                "),",
-                "anon_global_lvl AS (",
-                "  SELECT b.parent, b.child,",
-                "         MAX(CASE WHEN ar.allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
-                "         MAX(CASE WHEN ar.allow = 1 THEN 1 ELSE 0 END) AS any_allow",
-                "  FROM base b",
-                "  LEFT JOIN anon_rules ar ON ar.parent IS NULL AND ar.child IS NULL",
-                "  GROUP BY b.parent, b.child",
-                "),",
-                "anon_decisions AS (",
-                "  SELECT",
-                "    b.parent, b.child,",
-                "    CASE",
-                "      WHEN acl.any_deny = 1 THEN 0",
-                "      WHEN acl.any_allow = 1 THEN 1",
-                "      WHEN apl.any_deny = 1 THEN 0",
-                "      WHEN apl.any_allow = 1 THEN 1",
-                "      WHEN agl.any_deny = 1 THEN 0",
-                "      WHEN agl.any_allow = 1 THEN 1",
-                "      ELSE 0",
-                "    END AS anon_is_allowed",
-                "  FROM base b",
-                "  JOIN anon_child_lvl acl ON b.parent = acl.parent AND (b.child = acl.child OR (b.child IS NULL AND acl.child IS NULL))",
-                "  JOIN anon_parent_lvl apl ON b.parent = apl.parent AND (b.child = apl.child OR (b.child IS NULL AND apl.child IS NULL))",
-                "  JOIN anon_global_lvl agl ON b.parent = agl.parent AND (b.child = agl.child OR (b.child IS NULL AND agl.child IS NULL))",
-                "),",
+
+        def _anon_agg(select_key, where, group_by):
+            parts = [
+                f"  SELECT {select_key}",
+                "         MAX(CASE WHEN allow = 0 THEN 1 ELSE 0 END) AS any_deny,",
+                "         MAX(CASE WHEN allow = 1 THEN 1 ELSE 0 END) AS any_allow",
+                f"  FROM anon_rules WHERE {where}",
             ]
+            if group_by:
+                parts.append(f"  GROUP BY {group_by}")
+            return parts
+
+        query_parts.extend(
+            ["anon_child_agg AS ("]
+            + _anon_agg(
+                "parent, child,",
+                "parent IS NOT NULL AND child IS NOT NULL",
+                "parent, child",
+            )
+            + ["),", "anon_parent_agg AS ("]
+            + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent")
+            + ["),", "anon_global_agg AS ("]
+            + _anon_agg("", "parent IS NULL AND child IS NULL", None)
+            + ["),"]
         )
 
     # Final decisions
@@ -340,31 +316,28 @@ async def _build_single_action_sql(
             "decisions AS (",
             "  SELECT",
             "    b.parent, b.child,",
-            "    -- Cascading permission logic: child → parent → global, DENY beats ALLOW at each level",
+            "    -- Cascading permission logic: child -> parent -> global, DENY beats ALLOW at each level",
             "    -- Priority order:",
-            "    --   1. Child-level deny (most specific, blocks access)",
-            "    --   2. Child-level allow (most specific, grants access)",
-            "    --   3. Parent-level deny (intermediate, blocks access)",
-            "    --   4. Parent-level allow (intermediate, grants access)",
-            "    --   5. Global-level deny (least specific, blocks access)",
-            "    --   6. Global-level allow (least specific, grants access)",
+            "    --   1. Child-level deny  2. Child-level allow",
+            "    --   3. Parent-level deny 4. Parent-level allow",
+            "    --   5. Global-level deny 6. Global-level allow",
             "    --   7. Default deny (no rules match)",
             "    CASE",
-            "      WHEN cl.any_deny = 1 THEN 0",
-            "      WHEN cl.any_allow = 1 THEN 1",
-            "      WHEN pl.any_deny = 1 THEN 0",
-            "      WHEN pl.any_allow = 1 THEN 1",
-            "      WHEN gl.any_deny = 1 THEN 0",
-            "      WHEN gl.any_allow = 1 THEN 1",
+            "      WHEN ca.any_deny = 1 THEN 0",
+            "      WHEN ca.any_allow = 1 THEN 1",
+            "      WHEN pa.any_deny = 1 THEN 0",
+            "      WHEN pa.any_allow = 1 THEN 1",
+            "      WHEN ga.any_deny = 1 THEN 0",
+            "      WHEN ga.any_allow = 1 THEN 1",
             "      ELSE 0",
             "    END AS is_allowed,",
             "    CASE",
-            "      WHEN cl.any_deny = 1 THEN cl.deny_reasons",
-            "      WHEN cl.any_allow = 1 THEN cl.allow_reasons",
-            "      WHEN pl.any_deny = 1 THEN pl.deny_reasons",
-            "      WHEN pl.any_allow = 1 THEN pl.allow_reasons",
-            "      WHEN gl.any_deny = 1 THEN gl.deny_reasons",
-            "      WHEN gl.any_allow = 1 THEN gl.allow_reasons",
+            "      WHEN ca.any_deny = 1 THEN ca.deny_reasons",
+            "      WHEN ca.any_allow = 1 THEN ca.allow_reasons",
+            "      WHEN pa.any_deny = 1 THEN pa.deny_reasons",
+            "      WHEN pa.any_allow = 1 THEN pa.allow_reasons",
+            "      WHEN ga.any_deny = 1 THEN ga.deny_reasons",
+            "      WHEN ga.any_allow = 1 THEN ga.allow_reasons",
             "      ELSE '[]'",
             "    END AS reason",
         ]
@@ -372,21 +345,34 @@ async def _build_single_action_sql(
 
     if include_is_private:
         query_parts.append(
-            "    , CASE WHEN ad.anon_is_allowed = 0 THEN 1 ELSE 0 END AS is_private"
+            "    , CASE WHEN ("
+            "CASE"
+            " WHEN aca.any_deny = 1 THEN 0"
+            " WHEN aca.any_allow = 1 THEN 1"
+            " WHEN apa.any_deny = 1 THEN 0"
+            " WHEN apa.any_allow = 1 THEN 1"
+            " WHEN aga.any_deny = 1 THEN 0"
+            " WHEN aga.any_allow = 1 THEN 1"
+            " ELSE 0 END"
+            ") = 0 THEN 1 ELSE 0 END AS is_private"
         )
 
     query_parts.extend(
         [
             "  FROM base b",
-            "  JOIN child_lvl cl ON b.parent = cl.parent AND (b.child = cl.child OR (b.child IS NULL AND cl.child IS NULL))",
-            "  JOIN parent_lvl pl ON b.parent = pl.parent AND (b.child = pl.child OR (b.child IS NULL AND pl.child IS NULL))",
-            "  JOIN global_lvl gl ON b.parent = gl.parent AND (b.child = gl.child OR (b.child IS NULL AND gl.child IS NULL))",
+            "  LEFT JOIN child_agg ca ON ca.parent = b.parent AND ca.child = b.child",
+            "  LEFT JOIN parent_agg pa ON pa.parent = b.parent",
+            "  CROSS JOIN global_agg ga",
         ]
     )
 
     if include_is_private:
-        query_parts.append(
-            "  JOIN anon_decisions ad ON b.parent = ad.parent AND (b.child = ad.child OR (b.child IS NULL AND ad.child IS NULL))"
+        query_parts.extend(
+            [
+                "  LEFT JOIN anon_child_agg aca ON aca.parent = b.parent AND aca.child = b.child",
+                "  LEFT JOIN anon_parent_agg apa ON apa.parent = b.parent",
+                "  CROSS JOIN anon_global_agg aga",
+            ]
         )
 
     query_parts.append(")")
@@ -398,8 +384,28 @@ async def _build_single_action_sql(
         restriction_intersect = "\nINTERSECT\n".join(
             f"SELECT * FROM ({sql})" for sql in restriction_sqls
         )
+        # Decompose by NULL-pattern so the final filter can use pure-equality
+        # EXISTS lookups (satisfiable via automatic indexes) instead of a
+        # correlated OR-scan over the whole list.
         query_parts.extend(
-            [",", "restriction_list AS (", f"  {restriction_intersect}", ")"]
+            [
+                ",",
+                "restriction_list AS (",
+                f"  {restriction_intersect}",
+                "),",
+                "restriction_exact AS (",
+                "  SELECT parent, child FROM restriction_list WHERE parent IS NOT NULL AND child IS NOT NULL",
+                "),",
+                "restriction_parent_any AS (",
+                "  SELECT DISTINCT parent FROM restriction_list WHERE parent IS NOT NULL AND child IS NULL",
+                "),",
+                "restriction_child_any AS (",
+                "  SELECT DISTINCT child FROM restriction_list WHERE parent IS NULL AND child IS NOT NULL",
+                "),",
+                "restriction_all AS (",
+                "  SELECT 1 AS matched FROM restriction_list WHERE parent IS NULL AND child IS NULL LIMIT 1",
+                ")",
+            ]
         )
 
     # Final SELECT
@@ -414,10 +420,11 @@ async def _build_single_action_sql(
     # Add restriction filter if there are restrictions
     if restriction_sqls:
         query_parts.append("""
-  AND EXISTS (
-    SELECT 1 FROM restriction_list r
-    WHERE (r.parent = decisions.parent OR r.parent IS NULL)
-      AND (r.child = decisions.child OR r.child IS NULL)
+  AND (
+    EXISTS (SELECT 1 FROM restriction_all)
+    OR EXISTS (SELECT 1 FROM restriction_parent_any r WHERE r.parent = decisions.parent)
+    OR EXISTS (SELECT 1 FROM restriction_child_any r WHERE r.child = decisions.child)
+    OR EXISTS (SELECT 1 FROM restriction_exact r WHERE r.parent = decisions.parent AND r.child = decisions.child)
   )""")
 
     # Add parent filter if specified
@@ -495,6 +502,153 @@ async def build_permission_rules_sql(
     return rules_union, all_params, restriction_sqls
 
 
+async def check_permissions_for_actions(
+    *,
+    datasette: "Datasette",
+    actor: dict | None,
+    actions: list[str],
+    parent: str | None,
+    child: str | None,
+) -> dict[str, bool]:
+    """
+    Check several actions for one actor and resource in a single query.
+
+    Args:
+        datasette: The Datasette instance
+        actor: The actor dict (or None)
+        actions: List of action names to check
+        parent: The parent resource identifier (e.g., database name, or None)
+        child: The child resource identifier (e.g., table name, or None)
+
+    Returns:
+        Dict mapping each action name to True (allowed) or False (denied)
+
+    Each action contributes its own tagged block of permission rules
+    (gathered from the permission_resources_sql hook, with parameters
+    namespaced per action to avoid collisions) plus an optional
+    restriction allowlist CTE. One internal database query resolves
+    the winning rule per action using the same specificity-then-deny
+    ordering as the rest of the permission system.
+
+    Note: this resolves each action independently - also_requires
+    dependencies are handled by the caller (Datasette.allowed_many).
+    """
+    from datasette.utils.permissions import SKIP_PERMISSION_CHECKS
+
+    for action in actions:
+        if not datasette.actions.get(action):
+            raise ValueError(f"Unknown action: {action}")
+
+    # Dedupe while preserving order
+    unique_actions = list(dict.fromkeys(actions))
+    if not unique_actions:
+        return {}
+
+    # Gather hook results for each action concurrently - hooks within a
+    # single action still run sequentially, preserving existing semantics
+    gathered = await asyncio.gather(
+        *(
+            gather_permission_sql_from_hooks(
+                datasette=datasette, actor=actor, action=action
+            )
+            for action in unique_actions
+        )
+    )
+
+    if any(result is SKIP_PERMISSION_CHECKS for result in gathered):
+        return {action: True for action in unique_actions}
+
+    params = {"_check_parent": parent, "_check_child": child}
+    ctes = []
+    result_rows = []
+    verdicts = {}
+
+    for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)):
+        prefix = f"a{i}_"
+        rule_parts = []
+        restriction_parts = []
+
+        for permission_sql in permission_sqls:
+            sql = permission_sql.sql
+            restriction_sql = permission_sql.restriction_sql
+            # Namespace this block's params so identical names used for
+            # different actions cannot collide
+            for key in permission_sql.params or {}:
+                new_key = prefix + key
+                params[new_key] = permission_sql.params[key]
+                pattern = re.compile(":" + re.escape(key) + r"(?![A-Za-z0-9_])")
+                if sql:
+                    sql = pattern.sub(":" + new_key, sql)
+                if restriction_sql:
+                    restriction_sql = pattern.sub(":" + new_key, restriction_sql)
+
+            if restriction_sql:
+                restriction_parts.append(restriction_sql)
+
+            # Skip plugins that only provide restriction_sql (no permission rules)
+            if sql is None:
+                continue
+            rule_parts.append(
+                f"SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
+            )
+
+        if not rule_parts:
+            # No rules from any plugin - default deny. Restrictions can
+            # only restrict, never grant, so no SQL is needed at all
+            verdicts[action] = False
+            continue
+        ctes.append(f"a{i}_rules AS (\n" + "\nUNION ALL\n".join(rule_parts) + "\n)")
+
+        # Winning rule for this action: most specific depth first, then
+        # deny-beats-allow, then source_plugin as a stable tie-break
+        verdict_sql = f"""COALESCE((
+  SELECT allow FROM (
+    SELECT allow, source_plugin,
+      CASE
+        WHEN child IS NOT NULL THEN 2
+        WHEN parent IS NOT NULL THEN 1
+        ELSE 0
+      END AS depth
+    FROM a{i}_rules
+    WHERE (parent IS NULL OR parent = :_check_parent)
+      AND (child IS NULL OR child = :_check_child)
+    ORDER BY
+      depth DESC,
+      CASE WHEN allow = 0 THEN 0 ELSE 1 END,
+      source_plugin
+    LIMIT 1
+  )
+), 0)"""
+
+        if restriction_parts:
+            # Database-level restrictions (parent, NULL) match all children
+            restriction_intersect = "\nINTERSECT\n".join(
+                f"SELECT * FROM ({sql})" for sql in restriction_parts
+            )
+            ctes.append(f"a{i}_restriction AS (\n{restriction_intersect}\n)")
+            verdict_sql = f"""({verdict_sql}) AND EXISTS (
+  SELECT 1 FROM a{i}_restriction r
+  WHERE (r.parent = :_check_parent OR r.parent IS NULL)
+    AND (r.child = :_check_child OR r.child IS NULL)
+)"""
+
+        result_rows.append(f"({i}, ({verdict_sql}))")
+
+    if result_rows:
+        ctes.append(
+            "results(action_idx, is_allowed) AS (VALUES\n"
+            + ",\n".join(result_rows)
+            + "\n)"
+        )
+        query = (
+            "WITH\n" + ",\n".join(ctes) + "\nSELECT action_idx, is_allowed FROM results"
+        )
+        result = await datasette.get_internal_database().execute(query, params)
+        for row in result.rows:
+            verdicts[unique_actions[row[0]]] = bool(row[1])
+    return verdicts
+
+
 async def check_permission_for_resource(
     *,
     datasette: "Datasette",
@@ -515,77 +669,248 @@ async def check_permission_for_resource(
 
     Returns:
         True if the actor is allowed, False otherwise
-
-    This builds the cascading permission query and checks if the specific
-    resource is in the allowed set.
     """
-    rules_union, all_params, restriction_sqls = await build_permission_rules_sql(
-        datasette, actor, action
+    results = await check_permissions_for_actions(
+        datasette=datasette,
+        actor=actor,
+        actions=[action],
+        parent=parent,
+        child=child,
+    )
+    return results[action]
+
+
+async def explain_permission_for_resource(
+    *,
+    datasette: "Datasette",
+    actor: dict | None,
+    action: str,
+    parent: str | None,
+    child: str | None,
+) -> dict:
+    """Explain a permission decision for one action and resource.
+
+    This is intended for Datasette's permission debugging tools. It uses the
+    same ``permission_resources_sql`` hook results and the same resolution
+    rules as :func:`check_permissions_for_actions`, but also returns the
+    matching rules, actor restriction results and ``also_requires`` chain.
+
+    The returned dictionary is part of Datasette's unstable debugging API.
+    """
+
+    action_obj = datasette.actions.get(action)
+    if action_obj is None:
+        raise ValueError(f"Unknown action: {action}")
+
+    explanation = await _explain_single_action(
+        datasette=datasette,
+        actor=actor,
+        action=action,
+        parent=parent,
+        child=child,
     )
 
-    # If no rules (empty SQL), default deny
-    if not rules_union:
-        return False
-
-    # Add parameters for the resource we're checking
-    all_params["_check_parent"] = parent
-    all_params["_check_child"] = child
-
-    # If there are restriction filters, check if the resource passes them first
-    if restriction_sqls:
-        # Check if resource is in restriction allowlist
-        # Database-level restrictions (parent, NULL) should match all children (parent, *)
-        # Wrap each restriction_sql in a subquery to avoid operator precedence issues
-        restriction_check = "\nINTERSECT\n".join(
-            f"SELECT * FROM ({sql})" for sql in restriction_sqls
+    required_actions = []
+    if action_obj.also_requires:
+        required = await explain_permission_for_resource(
+            datasette=datasette,
+            actor=actor,
+            action=action_obj.also_requires,
+            parent=parent,
+            child=child,
         )
-        restriction_query = f"""
-WITH restriction_list AS (
-    {restriction_check}
-)
-SELECT EXISTS (
-    SELECT 1 FROM restriction_list
-    WHERE (parent = :_check_parent OR parent IS NULL)
-      AND (child = :_check_child OR child IS NULL)
-) AS in_allowlist
-"""
-        result = await datasette.get_internal_database().execute(
-            restriction_query, all_params
+        required_actions.append(required)
+
+    explanation["required_actions"] = required_actions
+    explanation["allowed"] = bool(
+        explanation["rule_allowed"]
+        and explanation["restriction_allowed"]
+        and all(required["allowed"] for required in required_actions)
+    )
+    explanation["summary"] = _permission_explanation_summary(explanation)
+    return explanation
+
+
+async def _explain_single_action(
+    *,
+    datasette: "Datasette",
+    actor: dict | None,
+    action: str,
+    parent: str | None,
+    child: str | None,
+) -> dict:
+    """Return matching rules and restrictions for a single action."""
+    from datasette.utils.permissions import SKIP_PERMISSION_CHECKS
+
+    permission_sqls = await gather_permission_sql_from_hooks(
+        datasette=datasette,
+        actor=actor,
+        action=action,
+    )
+
+    if permission_sqls is SKIP_PERMISSION_CHECKS:
+        return {
+            "action": action,
+            "rule_allowed": True,
+            "restriction_allowed": True,
+            "winning_scope": "global",
+            "matched_rules": [
+                {
+                    "scope": "global",
+                    "effect": "allow",
+                    "source": "skip_permission_checks",
+                    "reason": "Permission checks were explicitly skipped",
+                    "decisive": True,
+                    "ignored_because": None,
+                }
+            ],
+            "restrictions": [],
+        }
+
+    db = datasette.get_internal_database()
+    matched_rules = []
+    restrictions = []
+
+    for permission_sql in permission_sqls:
+        params = dict(permission_sql.params or {})
+        parent_param = _unused_parameter_name(params, "_explain_parent")
+        params[parent_param] = parent
+        child_param = _unused_parameter_name(params, "_explain_child")
+        params[child_param] = child
+
+        if permission_sql.sql:
+            rows = await db.execute(
+                f"""
+                SELECT parent, child, allow, reason
+                FROM ({permission_sql.sql}) AS permission_rules
+                WHERE (parent IS NULL OR parent = :{parent_param})
+                  AND (child IS NULL OR child = :{child_param})
+                """,
+                params,
+            )
+            for row in rows:
+                specificity = (
+                    2
+                    if row["child"] is not None
+                    else 1 if row["parent"] is not None else 0
+                )
+                matched_rules.append(
+                    {
+                        "scope": ("resource", "parent", "global")[2 - specificity],
+                        "effect": "allow" if row["allow"] else "deny",
+                        "source": permission_sql.source,
+                        "reason": row["reason"],
+                        "_specificity": specificity,
+                    }
+                )
+
+        if permission_sql.restriction_sql:
+            restriction_row = (
+                await db.execute(
+                    f"""
+                    SELECT EXISTS(
+                        SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules
+                        WHERE (parent IS NULL OR parent = :{parent_param})
+                          AND (child IS NULL OR child = :{child_param})
+                    ) AS resource_is_in_allowlist
+                    """,
+                    params,
+                )
+            ).first()
+            restriction_allowed = bool(restriction_row[0])
+            restrictions.append(
+                {
+                    "source": permission_sql.source,
+                    "allowed": restriction_allowed,
+                    "reason": params.get("deny")
+                    or (
+                        "Resource is included in this restriction allowlist"
+                        if restriction_allowed
+                        else "Resource is not included in this restriction allowlist"
+                    ),
+                }
+            )
+
+    matched_rules.sort(
+        key=lambda rule: (
+            -rule["_specificity"],
+            0 if rule["effect"] == "deny" else 1,
+            rule["source"] or "",
+            rule["reason"] or "",
         )
-        if result.rows and not result.rows[0][0]:
-            # Resource not in restriction allowlist - deny
-            return False
+    )
 
-    query = f"""
-WITH
-all_rules AS (
-  {rules_union}
-),
-matched_rules AS (
-  SELECT ar.*,
-    CASE
-      WHEN ar.child IS NOT NULL THEN 2  -- child-level (most specific)
-      WHEN ar.parent IS NOT NULL THEN 1  -- parent-level
-      ELSE 0                             -- root/global
-    END AS depth
-  FROM all_rules ar
-  WHERE (ar.parent IS NULL OR ar.parent = :_check_parent)
-    AND (ar.child IS NULL OR ar.child = :_check_child)
-),
-winner AS (
-  SELECT *
-  FROM matched_rules
-  ORDER BY
-    depth DESC,                          -- specificity first (higher depth wins)
-    CASE WHEN allow=0 THEN 0 ELSE 1 END, -- then deny over allow
-    source_plugin                        -- stable tie-break
-  LIMIT 1
-)
-SELECT COALESCE((SELECT allow FROM winner), 0) AS is_allowed
-"""
+    if matched_rules:
+        winning_specificity = matched_rules[0]["_specificity"]
+        winning_rules = [
+            rule
+            for rule in matched_rules
+            if rule["_specificity"] == winning_specificity
+        ]
+        rule_allowed = not any(rule["effect"] == "deny" for rule in winning_rules)
+        winning_scope = winning_rules[0]["scope"]
+    else:
+        winning_specificity = None
+        rule_allowed = False
+        winning_scope = None
 
-    # Execute the query against the internal database
-    result = await datasette.get_internal_database().execute(query, all_params)
-    if result.rows:
-        return bool(result.rows[0][0])
-    return False
+    for rule in matched_rules:
+        specificity = rule.pop("_specificity")
+        if specificity != winning_specificity:
+            rule["decisive"] = False
+            rule["ignored_because"] = "A more specific rule matched"
+        elif not rule_allowed and rule["effect"] == "allow":
+            rule["decisive"] = False
+            rule["ignored_because"] = "A deny rule matched at the same scope"
+        else:
+            rule["decisive"] = True
+            rule["ignored_because"] = None
+
+    return {
+        "action": action,
+        "rule_allowed": rule_allowed,
+        "restriction_allowed": all(
+            restriction["allowed"] for restriction in restrictions
+        ),
+        "winning_scope": winning_scope,
+        "matched_rules": matched_rules,
+        "restrictions": restrictions,
+    }
+
+
+def _unused_parameter_name(params: dict, preferred: str) -> str:
+    """Return a SQL parameter name that is not already in ``params``."""
+    candidate = preferred
+    suffix = 2
+    while candidate in params:
+        candidate = f"{preferred}_{suffix}"
+        suffix += 1
+    return candidate
+
+
+def _permission_explanation_summary(explanation: dict) -> str:
+    denied_requirement = next(
+        (
+            required
+            for required in explanation["required_actions"]
+            if not required["allowed"]
+        ),
+        None,
+    )
+    if denied_requirement:
+        return (
+            f"Denied because {explanation['action']} also requires "
+            f"{denied_requirement['action']}, which was denied."
+        )
+    if not explanation["matched_rules"]:
+        return "Denied because no permission rule matched this actor and resource."
+    if not explanation["rule_allowed"]:
+        return (
+            f"Denied by a {explanation['winning_scope']}-level rule. "
+            "Deny rules take precedence over allow rules at the same scope."
+        )
+    if not explanation["restriction_allowed"]:
+        return (
+            "Denied because the resource is not included in the actor's restrictions."
+        )
+    return f"Allowed by the matching {explanation['winning_scope']}-level rule."

datasette/utils/asgi.py

@@ -1,6 +1,6 @@
 import json
 from typing import Optional
-from datasette.utils import MultiParams, calculate_etag
+from datasette.utils import MultiParams, calculate_etag, error_body, sha256_file
 from datasette.utils.multipart import (
     parse_form_data,
     MultipartParseError,
@@ -67,13 +67,25 @@ class BadRequest(Base400):
     status = 400
 
 
+class PayloadTooLarge(Base400):
+    status = 413
+
+
 SAMESITE_VALUES = ("strict", "lax", "none")
 
+# Bodies read fully into memory (post_body/post_vars/json) are capped at this
+# size unless the max_post_body_bytes setting says otherwise. Kept deliberately
+# far below multipart's DEFAULT_MAX_REQUEST_SIZE: that parser streams to disk,
+# while these bodies are held in RAM and json.loads() can multiply their
+# footprint several times over.
+DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024  # 2MB
+
 
 class Request:
-    def __init__(self, scope, receive):
+    def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES):
         self.scope = scope
         self.receive = receive
+        self.max_post_body_bytes = max_post_body_bytes
 
     def __repr__(self):
         return '<asgi.Request method="{}" url="{}">'.format(self.method, self.url)
@@ -141,20 +153,52 @@ class Request:
     def actor(self):
         return self.scope.get("actor", None)
 
-    async def post_body(self):
-        body = b""
+    async def post_body(self, max_bytes=None):
+        """
+        Read the request body fully into memory.
+
+        The body is capped at max_bytes - or self.max_post_body_bytes
+        (default 2MB, set from the max_post_body_bytes setting for requests
+        created by Datasette) if max_bytes is not provided. Pass max_bytes=0
+        to disable the limit. Raises PayloadTooLarge (HTTP 413) if exceeded -
+        oversized bodies are rejected as soon as the limit is passed, without
+        buffering the rest.
+        """
+        if max_bytes is None:
+            max_bytes = self.max_post_body_bytes
+        too_large = PayloadTooLarge(
+            "Request body exceeded maximum size of {} bytes".format(max_bytes)
+        )
+        if max_bytes:
+            # Reject early if the client declares an oversized body
+            try:
+                if int(self.headers.get("content-length", "")) > max_bytes:
+                    raise too_large
+            except ValueError:
+                # Missing or malformed - the streaming check below still applies
+                pass
+        chunks = []
+        received = 0
         more_body = True
         while more_body:
             message = await self.receive()
             assert message["type"] == "http.request", message
-            body += message.get("body", b"")
+            chunk = message.get("body", b"")
+            received += len(chunk)
+            if max_bytes and received > max_bytes:
+                raise too_large
+            chunks.append(chunk)
             more_body = message.get("more_body", False)
-        return body
+        return b"".join(chunks)
 
     async def post_vars(self):
         body = await self.post_body()
         return dict(parse_qsl(body.decode("utf-8"), keep_blank_values=True))
 
+    async def json(self):
+        body = await self.post_body()
+        return json.loads(body)
+
     async def form(
         self,
         files: bool = False,
@@ -393,6 +437,9 @@ async def asgi_send_file(
             )
 
 
+HASHED_STATIC_CACHE_CONTROL = "max-age=31536000, immutable, public"
+
+
 def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None):
     root_path = Path(root_path)
     static_headers = {}
@@ -419,11 +466,17 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None):
             return
         try:
             # Calculate ETag for filepath
+            hash_value = request.args.get("_hash")
+            if (
+                hash_value
+                and hash_value == sha256_file(full_path, chunk_size=chunk_size)[:12]
+            ):
+                headers["Cache-Control"] = HASHED_STATIC_CACHE_CONTROL
             etag = await calculate_etag(full_path, chunk_size=chunk_size)
             headers["ETag"] = etag
             if_none_match = request.headers.get("if-none-match")
             if if_none_match and if_none_match == etag:
-                return await asgi_send(send, "", 304)
+                return await asgi_send(send, "", 304, headers=headers)
             await asgi_send_file(
                 send, full_path, chunk_size=chunk_size, headers=headers
             )
@@ -522,6 +575,18 @@ class Response:
             content_type="application/json; charset=utf-8",
         )
 
+    @classmethod
+    def error(cls, messages, status=400, headers=None):
+        """
+        A JSON error response using Datasette's standard error format.
+
+        messages can be a single string or a list of strings. For errors
+        that should content-negotiate between JSON and HTML, raise
+        Forbidden, NotFound, BadRequest or DatasetteError instead and let
+        Datasette's error handling hooks build the response.
+        """
+        return cls.json(error_body(messages, status), status=status, headers=headers)
+
     @classmethod
     def redirect(cls, path, status=302, headers=None):
         headers = headers or {}

datasette/utils/internal_db.py

@@ -1,9 +1,30 @@
 import textwrap
-from datasette.utils import table_column_details
 
+from sqlite_utils import Database as SQLiteUtilsDatabase
+from sqlite_utils import Migrations
 
-async def init_internal_db(db):
-    create_tables_sql = textwrap.dedent("""
+from datasette.utils import escape_sqlite, table_column_details
+
+INTERNAL_DB_SCHEMA_TABLES = {
+    "catalog_databases",
+    "catalog_tables",
+    "catalog_views",
+    "catalog_columns",
+    "catalog_indexes",
+    "catalog_foreign_keys",
+    "metadata_instance",
+    "metadata_databases",
+    "metadata_resources",
+    "metadata_columns",
+    "column_types",
+    "queries",
+}
+
+INTERNAL_DB_SCHEMA_INDEXES = {
+    "queries_owner_idx",
+}
+
+INTERNAL_DB_SCHEMA_SQL = textwrap.dedent("""
     CREATE TABLE IF NOT EXISTS catalog_databases (
         database_name TEXT PRIMARY KEY,
         path TEXT,
@@ -67,99 +88,101 @@ async def init_internal_db(db):
         FOREIGN KEY (database_name) REFERENCES catalog_databases(database_name),
         FOREIGN KEY (database_name, table_name) REFERENCES catalog_tables(database_name, table_name)
     );
+
+    CREATE TABLE IF NOT EXISTS metadata_instance (
+        key text,
+        value text,
+        unique(key)
+    );
+
+    CREATE TABLE IF NOT EXISTS metadata_databases (
+        database_name text,
+        key text,
+        value text,
+        unique(database_name, key)
+    );
+
+    CREATE TABLE IF NOT EXISTS metadata_resources (
+        database_name text,
+        resource_name text,
+        key text,
+        value text,
+        unique(database_name, resource_name, key)
+    );
+
+    CREATE TABLE IF NOT EXISTS metadata_columns (
+        database_name text,
+        resource_name text,
+        column_name text,
+        key text,
+        value text,
+        unique(database_name, resource_name, column_name, key)
+    );
+
+    CREATE TABLE IF NOT EXISTS column_types (
+        database_name TEXT NOT NULL,
+        resource_name TEXT NOT NULL,
+        column_name TEXT NOT NULL,
+        column_type TEXT NOT NULL,
+        config TEXT,
+        PRIMARY KEY (database_name, resource_name, column_name)
+    );
+
+    CREATE TABLE IF NOT EXISTS queries (
+        database_name TEXT NOT NULL,
+        name TEXT NOT NULL,
+        sql TEXT NOT NULL,
+        title TEXT,
+        description TEXT,
+        description_html TEXT,
+        options TEXT NOT NULL DEFAULT '{}',
+        parameters TEXT NOT NULL DEFAULT '[]',
+        is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)),
+        is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)),
+        is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)),
+        source TEXT NOT NULL DEFAULT 'user',
+        owner_id TEXT,
+        created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+        PRIMARY KEY (database_name, name)
+    );
+
+    CREATE INDEX IF NOT EXISTS queries_owner_idx
+        ON queries(owner_id);
     """).strip()
-    await db.execute_write_script(create_tables_sql)
-    await initialize_metadata_tables(db)
 
 
-async def initialize_metadata_tables(db):
-    await db.execute_write_script(textwrap.dedent("""
-        CREATE TABLE IF NOT EXISTS metadata_instance (
-            key text,
-            value text,
-            unique(key)
-        );
-
-        CREATE TABLE IF NOT EXISTS metadata_databases (
-            database_name text,
-            key text,
-            value text,
-            unique(database_name, key)
-        );
-
-        CREATE TABLE IF NOT EXISTS metadata_resources (
-            database_name text,
-            resource_name text,
-            key text,
-            value text,
-            unique(database_name, resource_name, key)
-        );
-
-        CREATE TABLE IF NOT EXISTS metadata_columns (
-            database_name text,
-            resource_name text,
-            column_name text,
-            key text,
-            value text,
-            unique(database_name, resource_name, column_name, key)
-        );
-
-        CREATE TABLE IF NOT EXISTS column_types (
-            database_name TEXT NOT NULL,
-            resource_name TEXT NOT NULL,
-            column_name TEXT NOT NULL,
-            column_type TEXT NOT NULL,
-            config TEXT,
-            PRIMARY KEY (database_name, resource_name, column_name)
-        );
-
-        CREATE TABLE IF NOT EXISTS queries (
-            database_name TEXT NOT NULL,
-            name TEXT NOT NULL,
-            sql TEXT NOT NULL,
-            title TEXT,
-            description TEXT,
-            description_html TEXT,
-            options TEXT NOT NULL DEFAULT '{}',
-            parameters TEXT NOT NULL DEFAULT '[]',
-            is_write INTEGER NOT NULL DEFAULT 0 CHECK (is_write IN (0, 1)),
-            is_private INTEGER NOT NULL DEFAULT 0 CHECK (is_private IN (0, 1)),
-            is_trusted INTEGER NOT NULL DEFAULT 0 CHECK (is_trusted IN (0, 1)),
-            source TEXT NOT NULL DEFAULT 'user',
-            owner_id TEXT,
-            created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
-            updated_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
-            PRIMARY KEY (database_name, name)
-        );
-
-        CREATE INDEX IF NOT EXISTS queries_owner_idx
-            ON queries(owner_id);
-            """))
+internal_migrations = Migrations("datasette_internal")
 
 
-async def populate_schema_tables(internal_db, db):
+def _internal_schema_exists(db):
+    table_names = set(db.table_names())
+    if not INTERNAL_DB_SCHEMA_TABLES.issubset(table_names):
+        return False
+    index_names = {
+        row[0]
+        for row in db.execute("select name from sqlite_master where type = 'index'")
+    }
+    return INTERNAL_DB_SCHEMA_INDEXES.issubset(index_names)
+
+
+@internal_migrations(name="0001_initial")
+def initial_internal_schema(db):
+    if _internal_schema_exists(db):
+        return
+    db.executescript(INTERNAL_DB_SCHEMA_SQL)
+
+
+async def init_internal_db(db):
+    def apply_migrations(conn):
+        internal_migrations.apply(SQLiteUtilsDatabase(conn, execute_plugins=False))
+
+    await db.execute_write_fn(apply_migrations, transaction=False)
+
+
+async def populate_schema_tables(internal_db, db, schema_version):
     database_name = db.name
 
-    def delete_everything(conn):
-        conn.execute(
-            "DELETE FROM catalog_tables WHERE database_name = ?", [database_name]
-        )
-        conn.execute(
-            "DELETE FROM catalog_views WHERE database_name = ?", [database_name]
-        )
-        conn.execute(
-            "DELETE FROM catalog_columns WHERE database_name = ?", [database_name]
-        )
-        conn.execute(
-            "DELETE FROM catalog_foreign_keys WHERE database_name = ?",
-            [database_name],
-        )
-        conn.execute(
-            "DELETE FROM catalog_indexes WHERE database_name = ?", [database_name]
-        )
-
-    await internal_db.execute_write_fn(delete_everything)
-
     tables = (await db.execute("select * from sqlite_master WHERE type = 'table'")).rows
     views = (await db.execute("select * from sqlite_master WHERE type = 'view'")).rows
 
@@ -190,7 +213,7 @@ async def populate_schema_tables(internal_db, db):
                 for column in columns
             )
             foreign_keys = conn.execute(
-                f"PRAGMA foreign_key_list([{table_name}])"
+                f"PRAGMA foreign_key_list({escape_sqlite(table_name)})"
             ).fetchall()
             foreign_keys_to_insert.extend(
                 {
@@ -199,7 +222,9 @@ async def populate_schema_tables(internal_db, db):
                 }
                 for foreign_key in foreign_keys
             )
-            indexes = conn.execute(f"PRAGMA index_list([{table_name}])").fetchall()
+            indexes = conn.execute(
+                f"PRAGMA index_list({escape_sqlite(table_name)})"
+            ).fetchall()
             indexes_to_insert.extend(
                 {
                     **{"database_name": database_name, "table_name": table_name},
@@ -223,47 +248,76 @@ async def populate_schema_tables(internal_db, db):
         indexes_to_insert,
     ) = await db.execute_fn(collect_info)
 
-    await internal_db.execute_write_many(
-        """
-        INSERT INTO catalog_tables (database_name, table_name, rootpage, sql)
-        values (?, ?, ?, ?)
-    """,
-        tables_to_insert,
-    )
-    await internal_db.execute_write_many(
-        """
-        INSERT INTO catalog_views (database_name, view_name, rootpage, sql)
-        values (?, ?, ?, ?)
-    """,
-        views_to_insert,
-    )
-    await internal_db.execute_write_many(
-        """
-        INSERT INTO catalog_columns (
-            database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
-        ) VALUES (
-            :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
+    def replace_catalog(conn):
+        # Delete child rows before their catalog_tables parents so this also
+        # works if a prepare_connection plugin enables foreign key enforcement.
+        for table in (
+            "catalog_columns",
+            "catalog_foreign_keys",
+            "catalog_indexes",
+            "catalog_views",
+            "catalog_tables",
+        ):
+            conn.execute(
+                "DELETE FROM {} WHERE database_name = ?".format(table),
+                [database_name],
+            )
+        conn.execute(
+            """
+            INSERT OR REPLACE INTO catalog_databases (
+                database_name, path, is_memory, schema_version
+            ) VALUES (?, ?, ?, ?)
+            """,
+            [
+                database_name,
+                str(db.path) if db.path is not None else None,
+                db.is_memory,
+                schema_version,
+            ],
         )
-    """,
-        columns_to_insert,
-    )
-    await internal_db.execute_write_many(
-        """
-        INSERT INTO catalog_foreign_keys (
-            database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
-        ) VALUES (
-            :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
+        conn.executemany(
+            """
+            INSERT INTO catalog_tables (database_name, table_name, rootpage, sql)
+            values (?, ?, ?, ?)
+            """,
+            tables_to_insert,
         )
-    """,
-        foreign_keys_to_insert,
-    )
-    await internal_db.execute_write_many(
-        """
-        INSERT INTO catalog_indexes (
-            database_name, table_name, seq, name, "unique", origin, partial
-        ) VALUES (
-            :database_name, :table_name, :seq, :name, :unique, :origin, :partial
+        conn.executemany(
+            """
+            INSERT INTO catalog_views (database_name, view_name, rootpage, sql)
+            values (?, ?, ?, ?)
+            """,
+            views_to_insert,
         )
-    """,
-        indexes_to_insert,
-    )
+        conn.executemany(
+            """
+            INSERT INTO catalog_columns (
+                database_name, table_name, cid, name, type, "notnull", default_value, is_pk, hidden
+            ) VALUES (
+                :database_name, :table_name, :cid, :name, :type, :notnull, :default_value, :is_pk, :hidden
+            )
+            """,
+            columns_to_insert,
+        )
+        conn.executemany(
+            """
+            INSERT INTO catalog_foreign_keys (
+                database_name, table_name, "id", seq, "table", "from", "to", on_update, on_delete, match
+            ) VALUES (
+                :database_name, :table_name, :id, :seq, :table, :from, :to, :on_update, :on_delete, :match
+            )
+            """,
+            foreign_keys_to_insert,
+        )
+        conn.executemany(
+            """
+            INSERT INTO catalog_indexes (
+                database_name, table_name, seq, name, "unique", origin, partial
+            ) VALUES (
+                :database_name, :table_name, :seq, :name, :unique, :origin, :partial
+            )
+            """,
+            indexes_to_insert,
+        )
+
+    await internal_db.execute_write_fn(replace_catalog)

datasette/utils/sql_analysis.py

@@ -150,7 +150,6 @@ _SQLITE_INTERNAL_SCHEMA_FUNCTIONS = {
     "sqlite_rename_test",
     "substr",
 }
-
 _AUTHORIZER_ACTION_NAMES = {
     getattr(sqlite3, name): name
     for name in (
@@ -391,6 +390,10 @@ def analyze_sql_tables(
             )
             return sqlite3.SQLITE_OK
 
+        if action == sqlite3.SQLITE_RECURSIVE:
+            # Recursive CTE bookkeeping; table reads are reported separately.
+            return sqlite3.SQLITE_OK
+
         if action == sqlite3.SQLITE_FUNCTION and arg2 is not None:
             record(
                 "function",
@@ -485,17 +488,17 @@ def analyze_sql_tables(
         and key.operation in {"create", "alter", "drop"}
         for key in operations
     )
-    dropped_tables = {
+    dropped_tables_and_views = {
         (key.database, key.table)
         for key in operations
-        if key.operation == "drop" and key.target_type == "table"
+        if key.operation == "drop" and key.target_type in {"table", "view"}
     }
 
     def key_is_drop_table_delete(key: OperationKey) -> bool:
         return (
             key.operation == "delete"
             and key.target_type == "table"
-            and (key.database, key.table) in dropped_tables
+            and (key.database, key.table) in dropped_tables_and_views
         )
 
     has_user_table_access_in_schema_operation = any(

datasette/version.py

@@ -1,2 +1,2 @@
-__version__ = "1.0a33"
+__version__ = "1.0a37"
 __version_info__ = tuple(__version__.split("."))

datasette/views/__init__.py

@@ -1,2 +1,89 @@
+from dataclasses import dataclass
+import dataclasses
+import types
+import typing
+
+
+@dataclass(frozen=True)
+class ContextField:
+    name: str
+    type_name: str
+    help: str
+    from_extra: bool = False
+
+
+def _type_name(type_):
+    if type_ is type(None):
+        return "None"
+    origin = typing.get_origin(type_)
+    args = typing.get_args(type_)
+    if origin in (typing.Union, types.UnionType):
+        return " | ".join(_type_name(arg) for arg in args)
+    if origin is not None:
+        name = getattr(origin, "__name__", str(origin).removeprefix("typing."))
+        return "{}[{}]".format(name, ", ".join(_type_name(arg) for arg in args))
+    return getattr(type_, "__name__", str(type_).removeprefix("typing."))
+
+
+def from_extra():
+    """
+    Declare a Context dataclass field whose value comes from a registered
+    Extra of the same name - its documentation is the Extra description,
+    so the doc string lives next to the resolve() code rather than being
+    duplicated on the dataclass.
+    """
+    return dataclasses.field(metadata={"from_extra": True})
+
+
 class Context:
     "Base class for all documented contexts"
+
+    # Set on subclasses whose from_extra() fields should be resolved
+    # against the extras registry for this scope
+    extras_scope = None
+
+    @classmethod
+    def documented_fields(cls):
+        "List of ContextField describing the documented fields of this context"
+        documented = []
+        for f in dataclasses.fields(cls):
+            if f.name.startswith("_"):
+                continue
+            is_from_extra = bool(f.metadata.get("from_extra"))
+            if is_from_extra:
+                help_text = cls._extra_description(f.name)
+            else:
+                help_text = f.metadata.get("help", "")
+            documented.append(
+                ContextField(
+                    name=f.name,
+                    type_name=_type_name(f.type),
+                    help=help_text,
+                    from_extra=is_from_extra,
+                )
+            )
+        return documented
+
+    @classmethod
+    def _extra_description(cls, name):
+        # Imported lazily - table_extras is not needed just to define
+        # Context subclasses
+        from datasette.views.table_extras import table_extra_registry
+
+        try:
+            extra_class = table_extra_registry.classes_by_name[name]
+        except KeyError:
+            raise KeyError(
+                "{}.{} is declared with from_extra() but there is no "
+                "registered extra of that name".format(cls.__name__, name)
+            )
+        if cls.extras_scope is not None and not extra_class.available_for(
+            cls.extras_scope
+        ):
+            raise ValueError(
+                "{}.{} is declared with from_extra() but the {} extra is "
+                "not available for scope {}".format(
+                    cls.__name__, name, name, cls.extras_scope
+                )
+            )
+        return extra_class.description or ""

datasette/views/base.py

@@ -1,31 +1,19 @@
-import asyncio
 import csv
 import hashlib
 import sys
-import textwrap
-import time
-import urllib
-from markupsafe import escape
 
-
-from datasette.database import QueryInterrupted
 from datasette.utils.asgi import Request
 from datasette.utils import (
     add_cors_headers,
-    await_me_maybe,
     EscapeHtmlWriter,
     InvalidSql,
     LimitedWriter,
-    call_with_supported_arguments,
     path_from_row_pks,
-    path_with_added_args,
-    path_with_removed_args,
     path_with_format,
     sqlite3,
 )
 from datasette.utils.asgi import (
     AsgiStream,
-    NotFound,
     Response,
     BadRequest,
 )
@@ -40,12 +28,15 @@ class DatasetteError(Exception):
         status=500,
         template=None,
         message_is_html=False,
+        plain_message=None,
     ):
         self.message = message
         self.title = title
         self.error_dict = error_dict or {}
         self.status = status
         self.message_is_html = message_is_html
+        # Plain text used for JSON error responses when message is HTML
+        self.plain_message = plain_message
 
 
 class View:
@@ -61,9 +52,7 @@ class View:
             request.path.endswith(".json")
             or request.headers.get("content-type") == "application/json"
         ):
-            response = Response.json(
-                {"ok": False, "error": "Method not allowed"}, status=405
-            )
+            response = Response.error("Method not allowed", 405)
         else:
             response = Response.text("Method not allowed", status=405)
         return response
@@ -102,9 +91,7 @@ class BaseView:
             request.path.endswith(".json")
             or request.headers.get("content-type") == "application/json"
         ):
-            response = Response.json(
-                {"ok": False, "error": "Method not allowed"}, status=405
-            )
+            response = Response.error("Method not allowed", 405)
         else:
             response = Response.text("Method not allowed", status=405)
         return response
@@ -192,231 +179,6 @@ class BaseView:
         return view
 
 
-class DataView(BaseView):
-    name = ""
-
-    def redirect(self, request, path, forward_querystring=True, remove_args=None):
-        if request.query_string and "?" not in path and forward_querystring:
-            path = f"{path}?{request.query_string}"
-        if remove_args:
-            path = path_with_removed_args(request, remove_args, path=path)
-        r = Response.redirect(path)
-        r.headers["Link"] = f"<{path}>; rel=preload"
-        if self.ds.cors:
-            add_cors_headers(r.headers)
-        return r
-
-    async def data(self, request):
-        raise NotImplementedError
-
-    async def as_csv(self, request, database):
-        return await stream_csv(self.ds, self.data, request, database)
-
-    async def get(self, request):
-        db = await self.ds.resolve_database(request)
-        database = db.name
-        database_route = db.route
-
-        _format = request.url_vars["format"]
-        data_kwargs = {}
-
-        if _format == "csv":
-            return await self.as_csv(request, database_route)
-
-        if _format is None:
-            # HTML views default to expanding all foreign key labels
-            data_kwargs["default_labels"] = True
-
-        extra_template_data = {}
-        start = time.perf_counter()
-        status_code = None
-        templates = []
-        try:
-            response_or_template_contexts = await self.data(request, **data_kwargs)
-            if isinstance(response_or_template_contexts, Response):
-                return response_or_template_contexts
-            # If it has four items, it includes an HTTP status code
-            if len(response_or_template_contexts) == 4:
-                (
-                    data,
-                    extra_template_data,
-                    templates,
-                    status_code,
-                ) = response_or_template_contexts
-            else:
-                data, extra_template_data, templates = response_or_template_contexts
-        except QueryInterrupted as ex:
-            raise DatasetteError(
-                textwrap.dedent("""
-                <p>SQL query took too long. The time limit is controlled by the
-                <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>
-                configuration option.</p>
-                <textarea style="width: 90%">{}</textarea>
-                <script>
-                let ta = document.querySelector("textarea");
-                ta.style.height = ta.scrollHeight + "px";
-                </script>
-            """.format(escape(ex.sql))).strip(),
-                title="SQL Interrupted",
-                status=400,
-                message_is_html=True,
-            )
-        except (sqlite3.OperationalError, InvalidSql) as e:
-            raise DatasetteError(str(e), title="Invalid SQL", status=400)
-
-        except sqlite3.OperationalError as e:
-            raise DatasetteError(str(e))
-
-        except DatasetteError:
-            raise
-
-        end = time.perf_counter()
-        data["query_ms"] = (end - start) * 1000
-
-        # Special case for .jsono extension - redirect to _shape=objects
-        if _format == "jsono":
-            return self.redirect(
-                request,
-                path_with_added_args(
-                    request,
-                    {"_shape": "objects"},
-                    path=request.path.rsplit(".jsono", 1)[0] + ".json",
-                ),
-                forward_querystring=False,
-            )
-
-        if _format in self.ds.renderers.keys():
-            # Dispatch request to the correct output format renderer
-            # (CSV is not handled here due to streaming)
-            result = call_with_supported_arguments(
-                self.ds.renderers[_format][0],
-                datasette=self.ds,
-                columns=data.get("columns") or [],
-                rows=data.get("rows") or [],
-                sql=data.get("query", {}).get("sql", None),
-                query_name=data.get("query_name"),
-                database=database,
-                table=data.get("table"),
-                request=request,
-                view_name=self.name,
-                truncated=False,  # TODO: support this
-                error=data.get("error"),
-                # These will be deprecated in Datasette 1.0:
-                args=request.args,
-                data=data,
-            )
-            if asyncio.iscoroutine(result):
-                result = await result
-            if result is None:
-                raise NotFound("No data")
-            if isinstance(result, dict):
-                r = Response(
-                    body=result.get("body"),
-                    status=result.get("status_code", status_code or 200),
-                    content_type=result.get("content_type", "text/plain"),
-                    headers=result.get("headers"),
-                )
-            elif isinstance(result, Response):
-                r = result
-                if status_code is not None:
-                    # Over-ride the status code
-                    r.status = status_code
-            else:
-                assert False, f"{result} should be dict or Response"
-        else:
-            extras = {}
-            if callable(extra_template_data):
-                extras = extra_template_data()
-                if asyncio.iscoroutine(extras):
-                    extras = await extras
-            else:
-                extras = extra_template_data
-            url_labels_extra = {}
-            if data.get("expandable_columns"):
-                url_labels_extra = {"_labels": "on"}
-
-            renderers = {}
-            for key, (_, can_render) in self.ds.renderers.items():
-                it_can_render = call_with_supported_arguments(
-                    can_render,
-                    datasette=self.ds,
-                    columns=data.get("columns") or [],
-                    rows=data.get("rows") or [],
-                    sql=data.get("query", {}).get("sql", None),
-                    query_name=data.get("query_name"),
-                    database=database,
-                    table=data.get("table"),
-                    request=request,
-                    view_name=self.name,
-                )
-                it_can_render = await await_me_maybe(it_can_render)
-                if it_can_render:
-                    renderers[key] = self.ds.urls.path(
-                        path_with_format(
-                            request=request,
-                            path=request.scope.get("route_path"),
-                            format=key,
-                            extra_qs={**url_labels_extra},
-                        )
-                    )
-
-            url_csv_args = {"_size": "max", **url_labels_extra}
-            url_csv = self.ds.urls.path(
-                path_with_format(
-                    request=request,
-                    path=request.scope.get("route_path"),
-                    format="csv",
-                    extra_qs=url_csv_args,
-                )
-            )
-            url_csv_path = url_csv.split("?")[0]
-            context = {
-                **data,
-                **extras,
-                **{
-                    "renderers": renderers,
-                    "url_csv": url_csv,
-                    "url_csv_path": url_csv_path,
-                    "url_csv_hidden_args": [
-                        (key, value)
-                        for key, value in urllib.parse.parse_qsl(request.query_string)
-                        if key not in ("_labels", "_facet", "_size")
-                    ]
-                    + [("_size", "max")],
-                    "settings": self.ds.settings_dict(),
-                },
-            }
-            if "metadata" not in context:
-                context["metadata"] = await self.ds.get_instance_metadata()
-            r = await self.render(templates, request=request, context=context)
-            if status_code is not None:
-                r.status = status_code
-
-        ttl = request.args.get("_ttl", None)
-        if ttl is None or not ttl.isdigit():
-            ttl = self.ds.setting("default_cache_ttl")
-
-        return self.set_response_headers(r, ttl)
-
-    def set_response_headers(self, response, ttl):
-        # Set far-future cache expiry
-        if self.ds.cache_headers and response.status == 200:
-            ttl = int(ttl)
-            if ttl == 0:
-                ttl_header = "no-cache"
-            else:
-                ttl_header = f"max-age={ttl}"
-            response.headers["Cache-Control"] = ttl_header
-        response.headers["Referrer-Policy"] = "no-referrer"
-        if self.ds.cors:
-            add_cors_headers(response.headers)
-        return response
-
-
-def _error(messages, status=400):
-    return Response.json({"ok": False, "errors": messages}, status=status)
-
-
 async def stream_csv(datasette, fetch_data, request, database):
     kwargs = {}
     stream = request.args.get("_stream")

datasette/views/database.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from urllib.parse import parse_qsl, urlencode
 import asyncio
 import hashlib
@@ -6,19 +6,17 @@ import itertools
 import json
 import markupsafe
 import os
-import re
-import sqlite_utils
 import textwrap
 
-from datasette.events import AlterTableEvent, CreateTableEvent, InsertRowsEvent
-from datasette.extras import extra_names_from_request
+from datasette.extras import extra_names_from_request, ExtraScope
 from datasette.database import QueryInterrupted
 from datasette.resources import DatabaseResource, QueryResource
-from datasette.stored_queries import stored_query_to_dict
+from datasette.stored_queries import StoredQuery, stored_query_to_dict
 from datasette.write_sql import QueryWriteRejected
 from datasette.utils import (
     add_cors_headers,
     await_me_maybe,
+    error_body,
     call_with_supported_arguments,
     named_parameters as derive_named_parameters,
     format_bytes,
@@ -37,16 +35,40 @@ from datasette.utils import (
 from datasette.utils.asgi import AsgiFileDownload, NotFound, Response, Forbidden
 from datasette.plugins import pm
 
-from .base import BaseView, DatasetteError, View, _error, stream_csv
+from .base import DatasetteError, View, stream_csv
 from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns
 from .table_extras import (
     QueryExtraContext,
     resolve_query_extras,
     table_extra_registry,
 )
+from .table_create_alter import _create_table_ui_context
 from . import Context
 
 
+@dataclass
+class DatabaseTable:
+    "Summary of a table or view shown on database and query pages."
+
+    name: str
+    columns: list[str]
+    primary_keys: list[str]
+    count: int | None
+    count_truncated: bool
+    hidden: bool
+    fts_table: str | None
+    foreign_keys: dict[str, list[dict[str, str]]]
+    private: bool
+
+
+@dataclass
+class DatabaseViewInfo:
+    "Summary of a SQLite view shown on the database page."
+
+    name: str
+    private: bool
+
+
 class DatabaseView(View):
     async def get(self, request, datasette):
         format_ = request.url_vars.get("format") or "html"
@@ -96,7 +118,7 @@ class DatabaseView(View):
         # Filter to just views
         view_names_set = set(await db.view_names())
         sql_views = [
-            {"name": name, "private": allowed_dict[name].private}
+            DatabaseViewInfo(name=name, private=allowed_dict[name].private)
             for name in allowed_dict
             if name in view_names_set
         ]
@@ -117,8 +139,36 @@ class DatabaseView(View):
             else len(stored_queries)
         )
 
+        # Resolve the registered database-level actions for this database in
+        # one batched query, seeding the request permission cache so allowed()
+        # calls made inside plugin hooks below are served from the cache.
+        database_action_permissions = await datasette.allowed_many(
+            actions=[
+                name
+                for name, action in datasette.actions.items()
+                if action.resource_class is DatabaseResource
+            ],
+            resource=DatabaseResource(database),
+            actor=request.actor,
+        )
+        create_table_ui = await _create_table_ui_context(
+            datasette, request, db, database, database_action_permissions
+        )
+
         async def database_actions():
             links = []
+            if create_table_ui:
+                links.append(
+                    {
+                        "type": "button",
+                        "label": "Create table",
+                        "description": "Create a new table in this database.",
+                        "attrs": {
+                            "aria-label": "Create table in {}".format(database),
+                            "data-database-action": "create-table",
+                        },
+                    }
+                )
             for hook in pm.hook.database_actions(
                 datasette=datasette,
                 database=database,
@@ -143,9 +193,9 @@ class DatabaseView(View):
             "private": private,
             "path": datasette.urls.database(database),
             "size": db.size,
-            "tables": tables,
-            "hidden_count": len([t for t in tables if t["hidden"]]),
-            "views": sql_views,
+            "tables": [asdict(table) for table in tables],
+            "hidden_count": len([table for table in tables if table.hidden]),
+            "views": [asdict(view) for view in sql_views],
             "queries": [stored_query_to_dict(query) for query in stored_queries],
             "queries_more": queries_more,
             "queries_count": queries_count,
@@ -185,7 +235,7 @@ class DatabaseView(View):
                     path=datasette.urls.database(database),
                     size=db.size,
                     tables=tables,
-                    hidden_count=len([t for t in tables if t["hidden"]]),
+                    hidden_count=len([table for table in tables if table.hidden]),
                     views=sql_views,
                     queries=stored_queries,
                     queries_more=queries_more,
@@ -198,10 +248,12 @@ class DatabaseView(View):
                     ),
                     metadata=metadata,
                     database_color=db.color,
+                    database_page_data=(
+                        {"createTable": create_table_ui} if create_table_ui else {}
+                    ),
                     database_actions=database_actions,
                     show_hidden=request.args.get("_show_hidden"),
                     editable=True,
-                    count_limit=db.count_limit,
                     allow_download=datasette.setting("allow_download")
                     and not db.is_mutable
                     and not db.is_memory,
@@ -228,16 +280,32 @@ class DatabaseView(View):
 
 @dataclass
 class DatabaseContext(Context):
+    "The page listing the tables, views and queries in a database, e.g. /fixtures."
+
+    documented_template = "database.html"
+
     database: str = field(metadata={"help": "The name of the database"})
     private: bool = field(
         metadata={"help": "Boolean indicating if this is a private database"}
     )
     path: str = field(metadata={"help": "The URL path to this database"})
     size: int = field(metadata={"help": "The size of the database in bytes"})
-    tables: list = field(metadata={"help": "List of table objects in the database"})
+    tables: list[DatabaseTable] = field(
+        metadata={
+            "help": "List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total."
+        }
+    )
     hidden_count: int = field(metadata={"help": "Count of hidden tables"})
-    views: list = field(metadata={"help": "List of view objects in the database"})
-    queries: list = field(metadata={"help": "List of stored query objects"})
+    views: list[DatabaseViewInfo] = field(
+        metadata={
+            "help": "List of ``DatabaseViewInfo`` objects describing SQLite views in the database. Each item has ``name`` and ``private`` attributes."
+        }
+    )
+    queries: list[StoredQuery] = field(
+        metadata={
+            "help": "List of ``StoredQuery`` objects. Each has attributes including ``name``, ``sql``, ``title``, ``description``, ``description_html``, ``hide_sql``, ``fragment``, ``parameters``, ``is_write`` and ``private``."
+        }
+    )
     queries_more: bool = field(
         metadata={"help": "Boolean indicating if more stored queries are available"}
     )
@@ -246,45 +314,65 @@ class DatabaseContext(Context):
         metadata={"help": "Boolean indicating if custom SQL can be executed"}
     )
     table_columns: dict = field(
-        metadata={"help": "Dictionary mapping table names to their column lists"}
+        metadata={
+            "help": "Dictionary mapping table names to lists of column names, used to power SQL autocomplete."
+        }
+    )
+    metadata: dict = field(
+        metadata={
+            "help": "Metadata dictionary for the database, such as ``title``, ``description``, ``license`` and ``source`` values from Datasette metadata."
+        }
     )
-    metadata: dict = field(metadata={"help": "Metadata for the database"})
     database_color: str = field(metadata={"help": "The color assigned to the database"})
+    database_page_data: dict = field(
+        metadata={
+            "help": 'JSON data used by JavaScript on the database page. Currently ``{}`` or ``{"createTable": {...}}`` where ``createTable`` includes ``path``, ``foreignKeyTargetsPath``, ``databaseName``, ``columnTypes``, ``defaultExpressions``, ``canInsertRows`` and optional ``customColumnTypes``.'
+        }
+    )
     database_actions: callable = field(
         metadata={
-            "help": "Callable returning list of action links for the database menu"
+            "help": 'Async callable returning action items for the database menu. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions` and :ref:`plugin_hook_database_actions`.'
         }
     )
     show_hidden: str = field(metadata={"help": "Value of _show_hidden query parameter"})
     editable: bool = field(
         metadata={"help": "Boolean indicating if the database is editable"}
     )
-    count_limit: int = field(metadata={"help": "The maximum number of rows to count"})
     allow_download: bool = field(
         metadata={"help": "Boolean indicating if database download is allowed"}
     )
     attached_databases: list = field(
-        metadata={"help": "List of names of attached databases"}
+        metadata={
+            "help": "List of names of databases attached to this SQLite connection. This is only populated for the special ``/_memory`` database when Datasette is started with ``--crossdb`` for :ref:`cross_database_queries`."
+        }
     )
     alternate_url_json: str = field(
         metadata={"help": "URL for the alternate JSON version of this page"}
     )
     select_templates: list = field(
         metadata={
-            "help": "List of templates that were considered for rendering this page"
+            "help": "List of template names that were considered for this page, with the selected template prefixed by ``*``."
         }
     )
     top_database: callable = field(
-        metadata={"help": "Callable to render the top_database slot"}
+        metadata={
+            "help": "Async callable that renders the ``top_database`` plugin slot for this database and returns HTML."
+        }
     )
 
 
 @dataclass
 class QueryContext(Context):
+    "The page for arbitrary SQL queries (/database/-/query?sql=...) and stored queries (/database/query-name)."
+
+    documented_template = "query.html"
+
     database: str = field(metadata={"help": "The name of the database being queried"})
     database_color: str = field(metadata={"help": "The color of the database"})
     query: dict = field(
-        metadata={"help": "The SQL query object containing the `sql` string"}
+        metadata={
+            "help": "Dictionary describing the SQL query being executed, with ``sql`` and ``params`` keys."
+        }
     )
     stored_query: str = field(
         metadata={"help": "The name of the stored query if this is a stored query"}
@@ -301,7 +389,9 @@ class QueryContext(Context):
         }
     )
     metadata: dict = field(
-        metadata={"help": "Metadata about the database or the stored query"}
+        metadata={
+            "help": "Metadata dictionary for the database or stored query. Stored query metadata may include options such as ``hide_sql``, ``on_success_message`` and ``on_error_redirect``."
+        }
     )
     db_is_immutable: bool = field(
         metadata={"help": "Boolean indicating if this database is immutable"}
@@ -325,22 +415,44 @@ class QueryContext(Context):
     save_query_url: str = field(
         metadata={"help": "URL to save the current arbitrary SQL as a query"}
     )
-    tables: list = field(metadata={"help": "List of table objects in the database"})
+    tables: list[DatabaseTable] = field(
+        metadata={
+            "help": "List of ``DatabaseTable`` objects describing tables in the database. Each item has ``name``, ``columns``, ``primary_keys``, ``count``, ``count_truncated``, ``hidden``, ``fts_table``, ``foreign_keys`` and ``private`` attributes. ``count_truncated`` is true if ``count`` is a capped lower bound rather than an exact total."
+        }
+    )
     named_parameter_values: dict = field(
-        metadata={"help": "Dictionary of parameter names/values"}
+        metadata={
+            "help": "Dictionary of named SQL parameter values, keyed by parameter name without the leading ``:``."
+        }
     )
     edit_sql_url: str = field(
         metadata={"help": "URL to edit the SQL for a stored query"}
     )
-    display_rows: list = field(metadata={"help": "List of result rows to display"})
-    columns: list = field(metadata={"help": "List of column names"})
-    renderers: dict = field(metadata={"help": "Dictionary of renderer name to URL"})
+    display_rows: list = field(
+        metadata={
+            "help": "List of result rows formatted for HTML display. Each row is a list of rendered cell values in the same order as ``columns``."
+        }
+    )
+    columns: list = field(
+        metadata={
+            "help": "List of result column names in the order they appear in ``display_rows`` and ``rows``."
+        }
+    )
+    renderers: dict = field(
+        metadata={
+            "help": "Dictionary mapping output format names such as ``json`` to URLs for this query in that format."
+        }
+    )
     url_csv: str = field(metadata={"help": "URL for CSV export"})
     show_hide_hidden: str = field(
-        metadata={"help": "Hidden input field for the _show_sql parameter"}
+        metadata={
+            "help": "Rendered hidden ``<input>`` HTML preserving the current ``_hide_sql`` or ``_show_sql`` state."
+        }
     )
     table_columns: dict = field(
-        metadata={"help": "Dictionary of table name to list of column names"}
+        metadata={
+            "help": "Dictionary mapping table names to lists of column names, used to power SQL autocomplete."
+        }
     )
     alternate_url_json: str = field(
         metadata={"help": "URL for alternate JSON version of this page"}
@@ -348,23 +460,27 @@ class QueryContext(Context):
     # TODO: refactor this to somewhere else, probably ds.render_template()
     select_templates: list = field(
         metadata={
-            "help": "List of templates that were considered for rendering this page"
+            "help": "List of template names that were considered for this page, with the selected template prefixed by ``*``."
         }
     )
     top_query: callable = field(
-        metadata={"help": "Callable to render the top_query slot"}
+        metadata={
+            "help": "Async callable that renders the ``top_query`` plugin slot for this query and returns HTML."
+        }
     )
     top_stored_query: callable = field(
-        metadata={"help": "Callable to render the top_stored_query slot"}
+        metadata={
+            "help": "Async callable that renders the ``top_stored_query`` plugin slot for stored queries and returns HTML."
+        }
     )
     query_actions: callable = field(
         metadata={
-            "help": "Callable returning a list of links for the query action menu"
+            "help": 'Async callable returning action items for the query menu. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions` and :ref:`plugin_hook_query_actions`.'
         }
     )
 
 
-async def get_tables(datasette, request, db, allowed_dict):
+async def get_tables(datasette, request, db, allowed_dict) -> list[DatabaseTable]:
     """
     Get list of tables with metadata for the database view.
 
@@ -385,21 +501,36 @@ async def get_tables(datasette, request, db, allowed_dict):
 
         table_columns = await db.table_columns(table)
         tables.append(
-            {
-                "name": table,
-                "columns": table_columns,
-                "primary_keys": await db.primary_keys(table),
-                "count": table_counts[table],
-                "hidden": table in hidden_table_names,
-                "fts_table": await db.fts_table(table),
-                "foreign_keys": all_foreign_keys[table],
-                "private": allowed_dict[table].private,
-            }
+            DatabaseTable(
+                name=table,
+                columns=table_columns,
+                primary_keys=await db.primary_keys(table),
+                count=table_counts[table],
+                count_truncated=_table_count_truncated(
+                    datasette, db, table, table_counts[table]
+                ),
+                hidden=table in hidden_table_names,
+                fts_table=await db.fts_table(table),
+                foreign_keys=all_foreign_keys[table],
+                private=allowed_dict[table].private,
+            )
         )
-    tables.sort(key=lambda t: (t["hidden"], t["name"]))
+    tables.sort(key=lambda table: (table.hidden, table.name))
     return tables
 
 
+def _table_count_truncated(datasette, db, table, count):
+    if count != db.count_limit + 1:
+        return False
+    if not db.is_mutable and datasette.inspect_data:
+        try:
+            datasette.inspect_data[db.name]["tables"][table]["count"]
+            return False
+        except KeyError:
+            pass
+    return True
+
+
 async def database_download(request, datasette):
     from datasette.resources import DatabaseResource
 
@@ -477,11 +608,7 @@ class QueryView(View):
                 "_json"
             ):
                 return Response.json(
-                    {
-                        "ok": False,
-                        "message": ex.message,
-                        "redirect": None,
-                    },
+                    dict(error_body([ex.message], 403), redirect=None),
                     status=403,
                 )
             datasette.add_message(request, ex.message, datasette.ERROR)
@@ -516,8 +643,15 @@ class QueryView(View):
         ok = None
         redirect_url = None
         try:
+            execute_write_kwargs = {"request": request}
+            if stored_query.is_trusted:
+                analysis = await db.analyze_sql(stored_query.sql, params_for_query)
+                if any(
+                    operation.operation == "vacuum" for operation in analysis.operations
+                ):
+                    execute_write_kwargs["transaction"] = False
             cursor = await db.execute_write(
-                stored_query.sql, params_for_query, request=request
+                stored_query.sql, params_for_query, **execute_write_kwargs
             )
             # success message can come from on_success_message or on_success_message_sql
             message = None
@@ -551,12 +685,17 @@ class QueryView(View):
             redirect_url = stored_query.on_error_redirect
             ok = False
         if should_return_json:
+            if ok:
+                return Response.json(
+                    {
+                        "ok": True,
+                        "message": message,
+                        "redirect": redirect_url,
+                    }
+                )
             return Response.json(
-                {
-                    "ok": ok,
-                    "message": message,
-                    "redirect": redirect_url,
-                }
+                dict(error_body([message], 400), redirect=redirect_url),
+                status=400,
             )
         else:
             datasette.add_message(request, message, message_type)
@@ -687,6 +826,10 @@ class QueryView(View):
                     title="SQL Interrupted",
                     status=400,
                     message_is_html=True,
+                    plain_message=(
+                        "SQL query took too long. The time limit is"
+                        " controlled by the sql_time_limit_ms setting."
+                    ),
                 )
             except sqlite3.DatabaseError as ex:
                 query_error = str(ex)
@@ -719,8 +862,11 @@ class QueryView(View):
 
             return await stream_csv(datasette, fetch_data_for_csv, request, db.name)
         elif format_ in datasette.renderers.keys():
+            if not sql:
+                raise DatasetteError("?sql= is required", status=400)
             data = {"ok": True, "rows": rows, "columns": columns}
             extras = extra_names_from_request(request)
+            table_extra_registry.validate_requested(extras, ExtraScope.QUERY)
             if extras:
                 query_extra_context = QueryExtraContext(
                     datasette=datasette,
@@ -1042,261 +1188,6 @@ class MagicParameters(dict):
             return super().__getitem__(key)
 
 
-class TableCreateView(BaseView):
-    name = "table-create"
-
-    _valid_keys = {
-        "table",
-        "rows",
-        "row",
-        "columns",
-        "pk",
-        "pks",
-        "ignore",
-        "replace",
-        "alter",
-    }
-    _supported_column_types = {
-        "text",
-        "integer",
-        "float",
-        "blob",
-    }
-    # Any string that does not contain a newline or start with sqlite_
-    _table_name_re = re.compile(r"^(?!sqlite_)[^\n]+$")
-
-    def __init__(self, datasette):
-        self.ds = datasette
-
-    async def post(self, request):
-        db = await self.ds.resolve_database(request)
-        database_name = db.name
-
-        # Must have create-table permission
-        if not await self.ds.allowed(
-            action="create-table",
-            resource=DatabaseResource(database=database_name),
-            actor=request.actor,
-        ):
-            return _error(["Permission denied"], 403)
-
-        body = await request.post_body()
-        try:
-            data = json.loads(body)
-        except json.JSONDecodeError as e:
-            return _error(["Invalid JSON: {}".format(e)])
-
-        if not isinstance(data, dict):
-            return _error(["JSON must be an object"])
-
-        invalid_keys = set(data.keys()) - self._valid_keys
-        if invalid_keys:
-            return _error(["Invalid keys: {}".format(", ".join(invalid_keys))])
-
-        # ignore and replace are mutually exclusive
-        if data.get("ignore") and data.get("replace"):
-            return _error(["ignore and replace are mutually exclusive"])
-
-        # ignore and replace only allowed with row or rows
-        if "ignore" in data or "replace" in data:
-            if not data.get("row") and not data.get("rows"):
-                return _error(["ignore and replace require row or rows"])
-
-        # ignore and replace require pk or pks
-        if "ignore" in data or "replace" in data:
-            if not data.get("pk") and not data.get("pks"):
-                return _error(["ignore and replace require pk or pks"])
-
-        ignore = data.get("ignore")
-        replace = data.get("replace")
-
-        if replace:
-            # Must have update-row permission
-            if not await self.ds.allowed(
-                action="update-row",
-                resource=DatabaseResource(database=database_name),
-                actor=request.actor,
-            ):
-                return _error(["Permission denied: need update-row"], 403)
-
-        table_name = data.get("table")
-        if not table_name:
-            return _error(["Table is required"])
-
-        if not self._table_name_re.match(table_name):
-            return _error(["Invalid table name"])
-
-        table_exists = await db.table_exists(data["table"])
-        columns = data.get("columns")
-        rows = data.get("rows")
-        row = data.get("row")
-        if not columns and not rows and not row:
-            return _error(["columns, rows or row is required"])
-
-        if rows and row:
-            return _error(["Cannot specify both rows and row"])
-
-        if rows or row:
-            # Must have insert-row permission
-            if not await self.ds.allowed(
-                action="insert-row",
-                resource=DatabaseResource(database=database_name),
-                actor=request.actor,
-            ):
-                return _error(["Permission denied: need insert-row"], 403)
-
-        alter = False
-        if rows or row:
-            if not table_exists:
-                # if table is being created for the first time, alter=True
-                alter = True
-            else:
-                # alter=True only if they request it AND they have permission
-                if data.get("alter"):
-                    if not await self.ds.allowed(
-                        action="alter-table",
-                        resource=DatabaseResource(database=database_name),
-                        actor=request.actor,
-                    ):
-                        return _error(["Permission denied: need alter-table"], 403)
-                    alter = True
-
-        if columns:
-            if rows or row:
-                return _error(["Cannot specify columns with rows or row"])
-            if not isinstance(columns, list):
-                return _error(["columns must be a list"])
-            for column in columns:
-                if not isinstance(column, dict):
-                    return _error(["columns must be a list of objects"])
-                if not column.get("name") or not isinstance(column.get("name"), str):
-                    return _error(["Column name is required"])
-                if not column.get("type"):
-                    column["type"] = "text"
-                if column["type"] not in self._supported_column_types:
-                    return _error(
-                        ["Unsupported column type: {}".format(column["type"])]
-                    )
-            # No duplicate column names
-            dupes = {c["name"] for c in columns if columns.count(c) > 1}
-            if dupes:
-                return _error(["Duplicate column name: {}".format(", ".join(dupes))])
-
-        if row:
-            rows = [row]
-
-        if rows:
-            if not isinstance(rows, list):
-                return _error(["rows must be a list"])
-            for row in rows:
-                if not isinstance(row, dict):
-                    return _error(["rows must be a list of objects"])
-
-        pk = data.get("pk")
-        pks = data.get("pks")
-
-        if pk and pks:
-            return _error(["Cannot specify both pk and pks"])
-        if pk:
-            if not isinstance(pk, str):
-                return _error(["pk must be a string"])
-        if pks:
-            if not isinstance(pks, list):
-                return _error(["pks must be a list"])
-            for pk in pks:
-                if not isinstance(pk, str):
-                    return _error(["pks must be a list of strings"])
-
-        # If table exists already, read pks from that instead
-        if table_exists:
-            actual_pks = await db.primary_keys(table_name)
-            # if pk passed and table already exists check it does not change
-            bad_pks = False
-            if len(actual_pks) == 1 and data.get("pk") and data["pk"] != actual_pks[0]:
-                bad_pks = True
-            elif (
-                len(actual_pks) > 1
-                and data.get("pks")
-                and set(data["pks"]) != set(actual_pks)
-            ):
-                bad_pks = True
-            if bad_pks:
-                return _error(["pk cannot be changed for existing table"])
-            pks = actual_pks
-
-        initial_schema = None
-        if table_exists:
-            initial_schema = await db.execute_fn(
-                lambda conn: sqlite_utils.Database(conn)[table_name].schema
-            )
-
-        def create_table(conn):
-            table = sqlite_utils.Database(conn)[table_name]
-            if rows:
-                table.insert_all(
-                    rows, pk=pks or pk, ignore=ignore, replace=replace, alter=alter
-                )
-            else:
-                table.create(
-                    {c["name"]: c["type"] for c in columns},
-                    pk=pks or pk,
-                )
-            return table.schema
-
-        try:
-            schema = await db.execute_write_fn(create_table, request=request)
-        except Exception as e:
-            return _error([str(e)])
-
-        if initial_schema is not None and initial_schema != schema:
-            await self.ds.track_event(
-                AlterTableEvent(
-                    request.actor,
-                    database=database_name,
-                    table=table_name,
-                    before_schema=initial_schema,
-                    after_schema=schema,
-                )
-            )
-
-        table_url = self.ds.absolute_url(
-            request, self.ds.urls.table(db.name, table_name)
-        )
-        table_api_url = self.ds.absolute_url(
-            request, self.ds.urls.table(db.name, table_name, format="json")
-        )
-        details = {
-            "ok": True,
-            "database": db.name,
-            "table": table_name,
-            "table_url": table_url,
-            "table_api_url": table_api_url,
-            "schema": schema,
-        }
-        if rows:
-            details["row_count"] = len(rows)
-
-        if not table_exists:
-            # Only log creation if we created a table
-            await self.ds.track_event(
-                CreateTableEvent(
-                    request.actor, database=db.name, table=table_name, schema=schema
-                )
-            )
-        if rows:
-            await self.ds.track_event(
-                InsertRowsEvent(
-                    request.actor,
-                    database=db.name,
-                    table=table_name,
-                    num_rows=len(rows),
-                    ignore=ignore,
-                    replace=replace,
-                )
-            )
-        return Response.json(details, status=201)
-
-
 async def display_rows(datasette, database, request, rows, columns):
     display_rows = []
     truncate_cells = datasette.setting("truncate_cells_html")

datasette/views/execute_write.py

@@ -2,10 +2,10 @@ import re
 from urllib.parse import urlencode
 
 from datasette.resources import DatabaseResource
-from datasette.utils import sqlite3
+from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3
 from datasette.utils.asgi import Response
 
-from .base import BaseView, _error
+from .base import BaseView
 from .database import display_rows as display_query_rows
 from .query_helpers import (
     QueryValidationError,
@@ -31,6 +31,15 @@ WRITE_TEMPLATE_LABELS = {
     "delete": "Delete rows",
 }
 WRITE_TEMPLATE_OPERATIONS = tuple(WRITE_TEMPLATE_LABELS)
+CREATE_TABLE_TEMPLATE_SQL = "\n".join(
+    (
+        "create table new_table (",
+        "  id integer primary key,",
+        "  name text",
+        "  -- created text default (datetime('now'))",
+        ")",
+    )
+)
 
 
 def _parameter_names(columns):
@@ -207,6 +216,23 @@ def _write_template_operations(write_template_tables):
     return operations
 
 
+async def _create_table_template_sql(datasette, db, actor):
+    if await datasette.allowed(
+        action="create-table",
+        resource=DatabaseResource(db.name),
+        actor=actor,
+    ):
+        return CREATE_TABLE_TEMPLATE_SQL
+    return None
+
+
+def _analysis_changes_schema(analysis):
+    return any(
+        operation.operation in {"create", "alter", "drop"}
+        for operation in analysis.operations
+    )
+
+
 class ExecuteWriteView(BaseView):
     name = "execute-write"
     has_json_alternate = False
@@ -241,6 +267,9 @@ class ExecuteWriteView(BaseView):
             self.ds, db, table_columns, hidden_table_names, request.actor
         )
         write_template_operations = _write_template_operations(write_template_tables)
+        write_create_table_template_sql = await _create_table_template_sql(
+            self.ds, db, request.actor
+        )
         if sql and analysis_error is None:
             try:
                 parameter_names = _derived_query_parameters(sql)
@@ -302,6 +331,7 @@ class ExecuteWriteView(BaseView):
                 "table_columns": table_columns,
                 "write_template_tables": write_template_tables,
                 "write_template_operations": write_template_operations,
+                "write_create_table_template_sql": write_create_table_template_sql,
                 "save_query_url": save_query_url,
                 "save_query_base_url": save_query_base_url,
             },
@@ -318,7 +348,7 @@ class ExecuteWriteView(BaseView):
         )
         if not db.is_mutable:
             return _block_framing(
-                _error(
+                Response.error(
                     ["Cannot execute write SQL because this database is immutable."],
                     403,
                 )
@@ -337,10 +367,10 @@ class ExecuteWriteView(BaseView):
             actor=request.actor,
         ):
             return _block_framing(
-                _error(["Permission denied: need execute-write-sql"], 403)
+                Response.error(["Permission denied: need execute-write-sql"], 403)
             )
         if not db.is_mutable:
-            return _block_framing(_error(["Database is immutable"], 403))
+            return _block_framing(Response.error(["Database is immutable"], 403))
 
         data = {}
         is_json = request.headers.get("content-type", "").startswith("application/json")
@@ -354,7 +384,7 @@ class ExecuteWriteView(BaseView):
             )
         except QueryValidationError as ex:
             if _wants_json(request, is_json, data):
-                return _block_framing(_error([ex.message], ex.status))
+                return _block_framing(Response.error([ex.message], ex.status))
             if ex.flash:
                 self.ds.add_message(request, ex.message, self.ds.ERROR)
             return await self._render_form(
@@ -375,7 +405,7 @@ class ExecuteWriteView(BaseView):
         except sqlite3.DatabaseError as ex:
             message = str(ex)
             if wants_json:
-                return _block_framing(_error([message], 400))
+                return _block_framing(Response.error([message], 400))
             return await self._render_form(
                 request,
                 db,
@@ -387,6 +417,9 @@ class ExecuteWriteView(BaseView):
                 status=400,
             )
 
+        if _analysis_changes_schema(analysis):
+            await self.ds.refresh_schemas(force=True)
+
         if cursor.rowcount == -1:
             message = "Query executed"
         else:
@@ -455,20 +488,18 @@ class ExecuteWriteAnalyzeView(BaseView):
             actor=request.actor,
         ):
             return _block_framing(
-                _error(["Permission denied: need execute-write-sql"], 403)
+                Response.error(["Permission denied: need execute-write-sql"], 403)
             )
 
         invalid_keys = set(request.args) - {"sql"}
         if invalid_keys:
             return _block_framing(
-                _error(
+                Response.error(
                     ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))],
                     400,
                 )
             )
         sql = request.args.get("sql") or ""
-        return _block_framing(
-            Response.json(
-                await _execute_write_analysis_data(self.ds, db, sql, request.actor)
-            )
-        )
+        analysis = await _execute_write_analysis_data(self.ds, db, sql, request.actor)
+        analysis["unstable"] = UNSTABLE_API_MESSAGE
+        return _block_framing(Response.json(analysis))

datasette/views/index.py

@@ -6,6 +6,7 @@ from datasette.utils import (
     await_me_maybe,
     make_slot_function,
     CustomJSONEncoder,
+    UNSTABLE_API_MESSAGE,
 )
 from datasette.utils.asgi import Response
 from datasette.version import __version__
@@ -151,7 +152,9 @@ class IndexView(BaseView):
             return Response(
                 json.dumps(
                     {
-                        "databases": {db["name"]: db for db in databases},
+                        "ok": True,
+                        "unstable": UNSTABLE_API_MESSAGE,
+                        "databases": databases,
                         "metadata": await self.ds.get_instance_metadata(),
                     },
                     cls=CustomJSONEncoder,

datasette/views/query_helpers.py

@@ -13,6 +13,7 @@ from datasette.write_sql import (
     operation_is_write,
 )
 from datasette.utils import (
+    parse_size_limit,
     named_parameters as derive_named_parameters,
     escape_sqlite,
     path_from_row_pks,
@@ -32,7 +33,6 @@ _query_fields = {
     "hide_sql",
     "fragment",
     "parameters",
-    "params",
     "is_private",
     "on_success_message",
     "on_success_redirect",
@@ -94,13 +94,11 @@ def _as_optional_bool(value, name):
     raise QueryValidationError("{} must be 0 or 1".format(name))
 
 
-def _query_list_limit(value, default=50):
-    if value in (None, ""):
-        return default
+def _query_list_limit(value, default, maximum):
     try:
-        return min(max(1, int(value)), 1000)
+        return parse_size_limit(value, default, maximum)
     except ValueError as ex:
-        raise QueryValidationError("_size must be an integer") from ex
+        raise QueryValidationError(str(ex)) from ex
 
 
 def _derived_query_parameters(sql):
@@ -541,7 +539,7 @@ async def _prepare_query_create(datasette, request, db, data):
         raise QueryValidationError("Writable query fields require writable SQL")
 
     parameters = _coerce_query_parameters(
-        data.get("parameters", data.get("params")),
+        data.get("parameters"),
         derived,
     )
     return {
@@ -586,9 +584,9 @@ async def _prepare_query_update(datasette, request, db, existing: StoredQuery, u
             actor=request.actor,
         )
 
-    if "parameters" in update or "params" in update:
+    if "parameters" in update:
         parameters = _coerce_query_parameters(
-            update.get("parameters", update.get("params")),
+            update.get("parameters"),
             derived,
         )
     elif "sql" in update:

datasette/views/row.py

@@ -1,27 +1,398 @@
-from datasette.utils.asgi import NotFound, Forbidden, Response
+import asyncio
+import json
+import textwrap
+import time
+import urllib.parse
+from dataclasses import dataclass, field
+
+import markupsafe
+import sqlite_utils
+
+from datasette.utils.asgi import NotFound, Forbidden, PayloadTooLarge, Response
 from datasette.database import QueryInterrupted
 from datasette.events import UpdateRowEvent, DeleteRowEvent
 from datasette.resources import TableResource
-from .base import DataView, BaseView, _error
+from .base import BaseView, DatasetteError, stream_csv
 from datasette.utils import (
+    add_cors_headers,
     await_me_maybe,
+    call_with_supported_arguments,
+    CustomJSONEncoder,
     CustomRow,
+    decode_write_json_row,
+    InvalidSql,
     make_slot_function,
+    path_from_row_pks,
+    path_with_format,
+    path_with_removed_args,
     to_css_class,
     escape_sqlite,
+    sqlite3,
+    WriteJsonValueError,
 )
 from datasette.plugins import pm
-import json
-import markupsafe
-import sqlite_utils
-from datasette.extras import extra_names_from_request
-from .table import display_columns_and_rows
+from datasette.extras import extra_names_from_request, ExtraScope
+from . import Context, from_extra
+from .table import (
+    display_columns_and_rows,
+    _table_page_data,
+    row_label_from_label_column,
+)
 from .table_extras import RowExtraContext, resolve_row_extras, table_extra_registry
 
 
-class RowView(DataView):
+@dataclass
+class RowContext(Context):
+    "The page showing an individual row, e.g. /fixtures/facetable/1."
+
+    documented_template = "row.html"
+    extras_scope = ExtraScope.ROW
+
+    # Fields resolved by registered extras - their documentation comes
+    # from the description on each Extra class in table_extras.py
+    columns: list = from_extra()
+    database: str = from_extra()
+    database_color: str = from_extra()
+    foreign_key_tables: list = from_extra()
+    metadata: dict = from_extra()
+    primary_keys: list = from_extra()
+    private: bool = from_extra()
+    table: str = from_extra()
+
+    # Fields added by the view code
+    ok: bool = field(
+        metadata={"help": "True if the data for this page was retrieved without errors"}
+    )
+    rows: list = field(
+        metadata={
+            "help": "A single-item list containing this row as a dictionary mapping column name to raw value."
+        }
+    )
+    primary_key_values: list = field(
+        metadata={"help": "Values of the primary keys for this row, from the URL"}
+    )
+    query_ms: float = field(
+        metadata={
+            "help": "Time taken by the SQL queries for this page, in milliseconds"
+        }
+    )
+    display_columns: list = field(
+        metadata={
+            "help": "Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys."
+        }
+    )
+    display_rows: list = field(
+        metadata={
+            "help": "Rows formatted for the HTML table display. Each row is iterable and contains cell dictionaries with ``column``, ``value``, ``raw`` and ``value_type`` keys."
+        }
+    )
+    custom_table_templates: list = field(
+        metadata={
+            "help": "Custom template names that were considered for displaying this row's table, in lookup order."
+        }
+    )
+    row_actions: list = field(
+        metadata={
+            "help": 'Row actions made available by core and plugin hooks. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions` and :ref:`plugin_hook_row_actions`.'
+        }
+    )
+    row_mutation_ui: bool = field(
+        metadata={"help": "True if the row edit/delete JavaScript UI should be enabled"}
+    )
+    table_page_data: dict = field(
+        metadata={
+            "help": "JSON data used by JavaScript on the row page. Includes ``database``, ``table`` and ``tableUrl``, plus optional ``foreignKeys`` mapping column names to autocomplete URLs."
+        }
+    )
+    top_row: callable = field(
+        metadata={
+            "help": "Async callable that renders the ``top_row`` plugin slot for this row and returns HTML."
+        }
+    )
+    renderers: dict = field(
+        metadata={
+            "help": "Dictionary mapping output format names such as ``json`` to URLs for this row in that format."
+        }
+    )
+    url_csv: str = field(metadata={"help": "URL for the CSV export of this page"})
+    url_csv_path: str = field(metadata={"help": "Path portion of the CSV export URL"})
+    url_csv_hidden_args: list = field(
+        metadata={
+            "help": "List of ``(name, value)`` pairs for hidden form fields used by the CSV export form, preserving current options while forcing ``_size=max``."
+        }
+    )
+    settings: dict = field(
+        metadata={
+            "help": "Dictionary of Datasette's current settings, keyed by setting name."
+        }
+    )
+    select_templates: list = field(
+        metadata={
+            "help": "List of template names that were considered for this page, with the selected template prefixed by ``*``."
+        }
+    )
+    alternate_url_json: str = field(
+        metadata={"help": "URL for the JSON version of this page"}
+    )
+
+
+class RowView(BaseView):
     name = "row"
 
+    def redirect(self, request, path, forward_querystring=True, remove_args=None):
+        if request.query_string and "?" not in path and forward_querystring:
+            path = f"{path}?{request.query_string}"
+        if remove_args:
+            path = path_with_removed_args(request, remove_args, path=path)
+        response = Response.redirect(path)
+        response.headers["Link"] = f"<{path}>; rel=preload"
+        if self.ds.cors:
+            add_cors_headers(response.headers)
+        return response
+
+    async def as_csv(self, request, database):
+        return await stream_csv(self.ds, self.data, request, database)
+
+    async def get(self, request):
+        db = await self.ds.resolve_database(request)
+        database = db.name
+        database_route = db.route
+        format_ = request.url_vars.get("format") or "html"
+        data_kwargs = {}
+
+        if format_ == "csv":
+            return await self.as_csv(request, database_route)
+
+        if format_ == "html":
+            # HTML views default to expanding all foreign key labels
+            data_kwargs["default_labels"] = True
+
+        extra_template_data = {}
+        start = time.perf_counter()
+        status_code = None
+        templates = ()
+        try:
+            response_or_template_contexts = await self.data(request, **data_kwargs)
+            if isinstance(response_or_template_contexts, Response):
+                return response_or_template_contexts
+            # If it has four items, it includes an HTTP status code
+            if len(response_or_template_contexts) == 4:
+                (
+                    data,
+                    extra_template_data,
+                    templates,
+                    status_code,
+                ) = response_or_template_contexts
+            else:
+                data, extra_template_data, templates = response_or_template_contexts
+        except QueryInterrupted as ex:
+            raise DatasetteError(
+                textwrap.dedent("""
+                <p>SQL query took too long. The time limit is controlled by the
+                <a href="https://docs.datasette.io/en/stable/settings.html#sql-time-limit-ms">sql_time_limit_ms</a>
+                configuration option.</p>
+                <textarea style="width: 90%">{}</textarea>
+                <script>
+                let ta = document.querySelector("textarea");
+                ta.style.height = ta.scrollHeight + "px";
+                </script>
+            """.format(markupsafe.escape(ex.sql))).strip(),
+                title="SQL Interrupted",
+                status=400,
+                message_is_html=True,
+                plain_message=(
+                    "SQL query took too long. The time limit is"
+                    " controlled by the sql_time_limit_ms setting."
+                ),
+            )
+        except (sqlite3.OperationalError, InvalidSql) as e:
+            raise DatasetteError(str(e), title="Invalid SQL", status=400)
+        except sqlite3.OperationalError as e:
+            raise DatasetteError(str(e))
+        except DatasetteError:
+            raise
+
+        end = time.perf_counter()
+        data["query_ms"] = (end - start) * 1000
+
+        if format_ in self.ds.renderers.keys():
+            # Dispatch request to the correct output format renderer
+            # (CSV is not handled here due to streaming)
+            result = call_with_supported_arguments(
+                self.ds.renderers[format_][0],
+                datasette=self.ds,
+                columns=data.get("columns") or [],
+                rows=data.get("rows") or [],
+                sql=data.get("query", {}).get("sql", None),
+                query_name=data.get("query_name"),
+                database=database,
+                table=data.get("table"),
+                request=request,
+                view_name=self.name,
+                truncated=False,  # TODO: support this
+                error=data.get("error"),
+                # These will be deprecated in Datasette 1.0:
+                args=request.args,
+                data=data,
+            )
+            if asyncio.iscoroutine(result):
+                result = await result
+            if result is None:
+                raise NotFound("No data")
+            if isinstance(result, dict):
+                response = Response(
+                    body=result.get("body"),
+                    status=result.get("status_code", status_code or 200),
+                    content_type=result.get("content_type", "text/plain"),
+                    headers=result.get("headers"),
+                )
+            elif isinstance(result, Response):
+                response = result
+                if status_code is not None:
+                    # Over-ride the status code
+                    response.status = status_code
+            else:
+                assert False, f"{result} should be dict or Response"
+        elif format_ == "html":
+            response = await self.html(request, data, extra_template_data, templates)
+            if status_code is not None:
+                response.status = status_code
+        else:
+            raise NotFound("Invalid format: {}".format(format_))
+
+        ttl = request.args.get("_ttl", None)
+        if ttl is None or not ttl.isdigit():
+            ttl = self.ds.setting("default_cache_ttl")
+
+        return self.set_response_headers(response, ttl)
+
+    async def html(self, request, data, extra_template_data, templates):
+        extras = {}
+        if callable(extra_template_data):
+            extras = extra_template_data()
+            if asyncio.iscoroutine(extras):
+                extras = await extras
+        else:
+            extras = extra_template_data
+
+        url_labels_extra = {}
+        if data.get("expandable_columns"):
+            url_labels_extra = {"_labels": "on"}
+
+        renderers = {}
+        for key, (_, can_render) in self.ds.renderers.items():
+            it_can_render = call_with_supported_arguments(
+                can_render,
+                datasette=self.ds,
+                columns=data.get("columns") or [],
+                rows=data.get("rows") or [],
+                sql=data.get("query", {}).get("sql", None),
+                query_name=data.get("query_name"),
+                database=data.get("database"),
+                table=data.get("table"),
+                request=request,
+                view_name=self.name,
+            )
+            it_can_render = await await_me_maybe(it_can_render)
+            if it_can_render:
+                renderers[key] = self.ds.urls.path(
+                    path_with_format(
+                        request=request,
+                        path=request.scope.get("route_path"),
+                        format=key,
+                        extra_qs={**url_labels_extra},
+                    )
+                )
+
+        url_csv_args = {"_size": "max", **url_labels_extra}
+        url_csv = self.ds.urls.path(
+            path_with_format(
+                request=request,
+                path=request.scope.get("route_path"),
+                format="csv",
+                extra_qs=url_csv_args,
+            )
+        )
+        url_csv_path = url_csv.split("?")[0]
+        context = {**data, **extras}
+        if "metadata" not in context:
+            context["metadata"] = await self.ds.get_instance_metadata()
+
+        environment = self.ds.get_jinja_environment(request)
+        template = environment.select_template(templates)
+        alternate_url_json = self.ds.absolute_url(
+            request,
+            self.ds.urls.path(
+                path_with_format(
+                    request=request,
+                    path=request.scope.get("route_path"),
+                    format="json",
+                )
+            ),
+        )
+        return Response.html(
+            await self.ds.render_template(
+                template,
+                RowContext(
+                    columns=context["columns"],
+                    database=context["database"],
+                    database_color=context["database_color"],
+                    foreign_key_tables=context["foreign_key_tables"],
+                    metadata=context["metadata"],
+                    primary_keys=context["primary_keys"],
+                    private=context["private"],
+                    table=context["table"],
+                    ok=context["ok"],
+                    rows=context["rows"],
+                    primary_key_values=context["primary_key_values"],
+                    query_ms=context["query_ms"],
+                    display_columns=context["display_columns"],
+                    display_rows=context["display_rows"],
+                    custom_table_templates=context["custom_table_templates"],
+                    row_actions=context["row_actions"],
+                    row_mutation_ui=context["row_mutation_ui"],
+                    table_page_data=context["table_page_data"],
+                    top_row=context["top_row"],
+                    renderers=renderers,
+                    url_csv=url_csv,
+                    url_csv_path=url_csv_path,
+                    url_csv_hidden_args=[
+                        (key, value)
+                        for key, value in urllib.parse.parse_qsl(request.query_string)
+                        if key not in ("_labels", "_facet", "_size")
+                    ]
+                    + [("_size", "max")],
+                    settings=self.ds.settings_dict(),
+                    select_templates=[
+                        f"{'*' if template_name == template.name else ''}{template_name}"
+                        for template_name in templates
+                    ],
+                    alternate_url_json=alternate_url_json,
+                ),
+                request=request,
+                view_name=self.name,
+            ),
+            headers={
+                "Link": '<{}>; rel="alternate"; type="application/json+datasette"'.format(
+                    alternate_url_json
+                )
+            },
+        )
+
+    def set_response_headers(self, response, ttl):
+        # Set far-future cache expiry
+        if self.ds.cache_headers and response.status == 200:
+            ttl = int(ttl)
+            if ttl == 0:
+                ttl_header = "no-cache"
+            else:
+                ttl_header = f"max-age={ttl}"
+            response.headers["Cache-Control"] = ttl_header
+        response.headers["Referrer-Policy"] = "no-referrer"
+        if self.ds.cors:
+            add_cors_headers(response.headers)
+        return response
+
     async def data(self, request, default_labels=False):
         resolved = await self.ds.resolve_row(request)
         db = resolved.db
@@ -49,6 +420,7 @@ class RowView(DataView):
         pks = resolved.pks
 
         async def template_data():
+            is_table = await db.table_exists(table)
             # Reorder columns so primary keys come first
             pk_set = set(pks)
             pk_cols = [d for d in results.description if d[0] in pk_set]
@@ -117,7 +489,60 @@ class RowView(DataView):
                             "<strong>{}</strong>".format(cell["value"])
                         )
 
+            label_column = await db.label_column_for_table(table) if is_table else None
+            row_path = path_from_row_pks(rows[0], pks, False)
+            pk_path = path_from_row_pks(rows[0], pks, False, False)
+            row_label = row_label_from_label_column(expanded_rows[0], label_column)
+            for display_row in display_rows:
+                display_row.pk_path = pk_path
+                display_row.row_path = row_path
+                display_row.row_label = row_label
+
+            row_action_label = pk_path
+            if row_label and row_label != pk_path:
+                row_action_label = "{} {}".format(pk_path, row_label)
+
+            row_action_permissions = {}
+            if is_table and db.is_mutable:
+                row_action_permissions = await self.ds.allowed_many(
+                    actions=["update-row", "delete-row"],
+                    resource=TableResource(database=database, table=table),
+                    actor=request.actor,
+                )
+
             row_actions = []
+            if row_action_permissions.get("update-row"):
+                attrs = {
+                    "aria-label": "Edit row {}".format(row_action_label),
+                    "data-row": row_path,
+                    "data-row-action": "edit",
+                }
+                if row_label:
+                    attrs["data-row-label"] = row_label
+                row_actions.append(
+                    {
+                        "type": "button",
+                        "label": "Edit row",
+                        "description": "Open a dialog to edit this row.",
+                        "attrs": attrs,
+                    }
+                )
+            if row_action_permissions.get("delete-row"):
+                attrs = {
+                    "aria-label": "Delete row {}".format(row_action_label),
+                    "data-row": row_path,
+                    "data-row-action": "delete",
+                }
+                if row_label:
+                    attrs["data-row-label"] = row_label
+                row_actions.append(
+                    {
+                        "type": "button",
+                        "label": "Delete row",
+                        "description": "Open a confirmation dialog to delete this row.",
+                        "attrs": attrs,
+                    }
+                )
             for hook in pm.hook.row_actions(
                 datasette=self.ds,
                 actor=request.actor,
@@ -144,6 +569,17 @@ class RowView(DataView):
                     f"_table-row-{to_css_class(database)}-{to_css_class(table)}.html",
                     "_table.html",
                 ],
+                "row_mutation_ui": any(row_action_permissions.values()),
+                "table_page_data": await _table_page_data(
+                    datasette=self.ds,
+                    request=request,
+                    db=db,
+                    database_name=database,
+                    table_name=table,
+                    is_view=not is_table,
+                    table_insert_ui=None,
+                    table_alter_ui=None,
+                ),
                 "row_actions": row_actions,
                 "top_row": make_slot_function(
                     "top_row",
@@ -167,6 +603,9 @@ class RowView(DataView):
         }
 
         extras = extra_names_from_request(request)
+        if request.url_vars.get("format"):
+            # Data formats reject unknown extras; HTML ignores them
+            table_extra_registry.validate_requested(extras, ExtraScope.ROW)
 
         # Process extras
         row_extra_context = RowExtraContext(
@@ -249,17 +688,40 @@ class RowError(Exception):
         self.error = error
 
 
+ROW_FLASH_LABEL_MAX_LENGTH = 80
+
+
+def _truncated_row_flash_label(label):
+    label = " ".join(str(label).split())
+    if len(label) <= ROW_FLASH_LABEL_MAX_LENGTH:
+        return label
+    return label[: ROW_FLASH_LABEL_MAX_LENGTH - 1] + "\u2026"
+
+
+async def _row_flash_message(db, action, resolved, row=None):
+    pk_label = ", ".join(resolved.pk_values)
+    label_column = await db.label_column_for_table(resolved.table)
+    label = row_label_from_label_column(row or resolved.row, label_column)
+    if label:
+        label = _truncated_row_flash_label(label)
+    if label and label != pk_label:
+        return "{} row {} ({})".format(action, pk_label, label)
+    return "{} row {}".format(action, pk_label)
+
+
 async def _resolve_row_and_check_permission(datasette, request, permission):
     from datasette.app import DatabaseNotFound, TableNotFound, RowNotFound
 
     try:
         resolved = await datasette.resolve_row(request)
     except DatabaseNotFound as e:
-        return False, _error(["Database not found: {}".format(e.database_name)], 404)
+        return False, Response.error(
+            ["Database not found: {}".format(e.database_name)], 404
+        )
     except TableNotFound as e:
-        return False, _error(["Table not found: {}".format(e.table)], 404)
+        return False, Response.error(["Table not found: {}".format(e.table)], 404)
     except RowNotFound as e:
-        return False, _error(["Record not found: {}".format(e.pk_values)], 404)
+        return False, Response.error(["Record not found: {}".format(e.pk_values)], 404)
 
     # Ensure user has permission to delete this row
     if not await datasette.allowed(
@@ -267,7 +729,7 @@ async def _resolve_row_and_check_permission(datasette, request, permission):
         resource=TableResource(database=resolved.db.name, table=resolved.table),
         actor=request.actor,
     ):
-        return False, _error(["Permission denied"], 403)
+        return False, Response.error(["Permission denied"], 403)
 
     return True, resolved
 
@@ -292,7 +754,7 @@ class RowDeleteView(BaseView):
         try:
             await resolved.db.execute_write_fn(delete_row, request=request)
         except Exception as e:
-            return _error([str(e)], 500)
+            return Response.error([str(e)], 400)
 
         await self.ds.track_event(
             DeleteRowEvent(
@@ -303,6 +765,15 @@ class RowDeleteView(BaseView):
             )
         )
 
+        if request.args.get("_redirect_to_table"):
+            table_url = self.ds.urls.table(resolved.db.name, resolved.table)
+            self.ds.add_message(
+                request,
+                await _row_flash_message(resolved.db, "Deleted", resolved),
+                self.ds.INFO,
+            )
+            return Response.json({"ok": True, "redirect": str(table_url)}, status=200)
+
         return Response.json({"ok": True}, status=200)
 
 
@@ -319,22 +790,27 @@ class RowUpdateView(BaseView):
         if not ok:
             return resolved
 
-        body = await request.post_body()
         try:
-            data = json.loads(body)
+            data = await request.json()
         except json.JSONDecodeError as e:
-            return _error(["Invalid JSON: {}".format(e)])
+            return Response.error(["Invalid JSON: {}".format(e)])
+        except PayloadTooLarge as e:
+            return Response.error([str(e)], 413)
 
         if not isinstance(data, dict):
-            return _error(["JSON must be a dictionary"])
+            return Response.error(["JSON must be a dictionary"])
         if "update" not in data or not isinstance(data["update"], dict):
-            return _error(["JSON must contain an update dictionary"])
+            return Response.error(["JSON must contain an update dictionary"])
 
         invalid_keys = set(data.keys()) - {"update", "return", "alter"}
         if invalid_keys:
-            return _error(["Invalid keys: {}".format(", ".join(invalid_keys))])
+            return Response.error(["Invalid keys: {}".format(", ".join(invalid_keys))])
 
         update = data["update"]
+        try:
+            update = decode_write_json_row(update)
+        except WriteJsonValueError as e:
+            return Response.error([str(e)], 400)
 
         # Validate column types
         from datasette.views.table import _validate_column_types
@@ -343,7 +819,7 @@ class RowUpdateView(BaseView):
             self.ds, resolved.db.name, resolved.table, [update]
         )
         if ct_errors:
-            return _error(ct_errors, 400)
+            return Response.error(ct_errors, 400)
 
         alter = data.get("alter")
         if alter and not await self.ds.allowed(
@@ -351,7 +827,7 @@ class RowUpdateView(BaseView):
             resource=TableResource(database=resolved.db.name, table=resolved.table),
             actor=request.actor,
         ):
-            return _error(["Permission denied for alter-table"], 403)
+            return Response.error(["Permission denied for alter-table"], 403)
 
         def update_row(conn):
             sqlite_utils.Database(conn)[resolved.table].update(
@@ -361,14 +837,16 @@ class RowUpdateView(BaseView):
         try:
             await resolved.db.execute_write_fn(update_row, request=request)
         except Exception as e:
-            return _error([str(e)], 400)
+            return Response.error([str(e)], 400)
 
         result = {"ok": True}
+        returned_row = None
         if data.get("return"):
             results = await resolved.db.execute(
                 resolved.sql, resolved.params, truncate=True
             )
-            result["row"] = results.dicts()[0]
+            returned_row = results.dicts()[0]
+            result["rows"] = [returned_row]
 
         await self.ds.track_event(
             UpdateRowEvent(
@@ -379,4 +857,19 @@ class RowUpdateView(BaseView):
             )
         )
 
-        return Response.json(result, status=200)
+        if request.args.get("_message"):
+            message_row = returned_row
+            if message_row is None:
+                results = await resolved.db.execute(
+                    resolved.sql, resolved.params, truncate=True
+                )
+                message_row = results.first()
+            self.ds.add_message(
+                request,
+                await _row_flash_message(
+                    resolved.db, "Updated", resolved, row=message_row
+                ),
+                self.ds.INFO,
+            )
+
+        return Response.json(result, status=200, default=CustomJSONEncoder().default)

datasette/views/special.py

@@ -6,9 +6,12 @@ from datasette.events import LogoutEvent, LoginEvent, CreateTokenEvent
 from datasette.resources import DatabaseResource, TableResource
 from datasette.utils.asgi import Response, Forbidden
 from datasette.utils import (
+    UNSTABLE_API_MESSAGE,
     actor_matches_allow,
+    parse_size_limit,
     add_cors_headers,
     await_me_maybe,
+    error_body,
     tilde_encode,
     tilde_decode,
 )
@@ -52,9 +55,9 @@ class JsonDataView(BaseView):
         if self.permission:
             await self.ds.ensure_permission(action=self.permission, actor=request.actor)
         if self.needs_request:
-            data = self.data_callback(request)
+            data = await await_me_maybe(self.data_callback(request))
         else:
-            data = self.data_callback()
+            data = await await_me_maybe(self.data_callback())
 
         # Return JSON or HTML depending on format parameter
         as_format = request.url_vars.get("format")
@@ -62,6 +65,8 @@ class JsonDataView(BaseView):
             headers = {}
             if self.ds.cors:
                 add_cors_headers(headers)
+            if isinstance(data, dict):
+                data = {"ok": True, **data}
             return Response.json(data, headers=headers)
         else:
             context = {
@@ -91,6 +96,110 @@ class PatternPortfolioView(View):
         )
 
 
+class AutocompleteDebugView(BaseView):
+    name = "autocomplete_debug"
+    has_json_alternate = False
+
+    async def _suggested_tables(self, request):
+        scanned = 0
+        reached_scan_limit = False
+        suggestions = []
+        for database_name, db in self.ds.databases.items():
+            if scanned >= 100 or len(suggestions) >= 5:
+                break
+            remaining = 100 - scanned
+            results = await db.execute(
+                "select name from sqlite_master where type = 'table' order by name limit ?",
+                [remaining],
+            )
+            for row in results.rows:
+                table_name = row["name"]
+                scanned += 1
+                if scanned >= 100:
+                    reached_scan_limit = True
+                visible, _ = await self.ds.check_visibility(
+                    request.actor,
+                    action="view-table",
+                    resource=TableResource(database=database_name, table=table_name),
+                )
+                if not visible:
+                    if scanned >= 100:
+                        break
+                    continue
+                label_column = await db.label_column_for_table(table_name)
+                if label_column:
+                    suggestions.append(
+                        {
+                            "database": database_name,
+                            "table": table_name,
+                            "label_column": label_column,
+                            "url": self.ds.urls.path(
+                                "-/debug/autocomplete?"
+                                + urllib.parse.urlencode(
+                                    {
+                                        "database": database_name,
+                                        "table": table_name,
+                                    }
+                                )
+                            ),
+                        }
+                    )
+                    if len(suggestions) >= 5:
+                        break
+                if scanned >= 100:
+                    break
+        return suggestions, scanned, reached_scan_limit
+
+    async def get(self, request):
+        await self.ds.ensure_permission(action="view-instance", actor=request.actor)
+        database_name = request.args.get("database")
+        table_name = request.args.get("table")
+        context = {
+            "database_name": database_name,
+            "table_name": table_name,
+        }
+
+        if database_name or table_name:
+            if not database_name or not table_name:
+                context["error"] = "Both database and table are required."
+            elif database_name not in self.ds.databases:
+                context["error"] = "Database not found."
+            else:
+                db = self.ds.databases[database_name]
+                if not await db.table_exists(table_name):
+                    context["error"] = "Table not found."
+                else:
+                    await self.ds.ensure_permission(
+                        action="view-table",
+                        resource=TableResource(
+                            database=database_name,
+                            table=table_name,
+                        ),
+                        actor=request.actor,
+                    )
+                    context.update(
+                        {
+                            "autocomplete_url": "{}/-/autocomplete".format(
+                                self.ds.urls.table(database_name, table_name)
+                            ),
+                            "label_column": await db.label_column_for_table(table_name),
+                        }
+                    )
+        else:
+            suggestions, scanned, reached_scan_limit = await self._suggested_tables(
+                request
+            )
+            context.update(
+                {
+                    "suggestions": suggestions,
+                    "scanned": scanned,
+                    "reached_scan_limit": reached_scan_limit,
+                }
+            )
+
+        return await self.render(["debug_autocomplete.html"], request, context)
+
+
 class AuthTokenView(BaseView):
     name = "auth_token"
     has_json_alternate = False
@@ -188,6 +297,12 @@ class PermissionsDebugView(BaseView):
         response, status = await _check_permission_for_actor(
             self.ds, permission, parent, child, actor
         )
+        if response.get("ok"):
+            response = {
+                "ok": True,
+                "unstable": UNSTABLE_API_MESSAGE,
+                **response,
+            }
         return Response.json(response, status=status)
 
 
@@ -244,29 +359,32 @@ class AllowedResourcesView(BaseView):
     async def _allowed_payload(self, request, has_debug_permission):
         action = request.args.get("action")
         if not action:
-            return {"error": "action parameter is required"}, 400
+            return error_body("action parameter is required", 400), 400
         if action not in self.ds.actions:
-            return {"error": f"Unknown action: {action}"}, 404
+            return error_body(f"Unknown action: {action}", 404), 404
 
         actor = request.actor if isinstance(request.actor, dict) else None
         actor_id = actor.get("id") if actor else None
         parent_filter = request.args.get("parent")
         child_filter = request.args.get("child")
         if child_filter and not parent_filter:
-            return {"error": "parent must be provided when child is specified"}, 400
+            return (
+                error_body("parent must be provided when child is specified", 400),
+                400,
+            )
 
         try:
-            page = int(request.args.get("page", "1"))
-            page_size = int(request.args.get("page_size", "50"))
+            page = int(request.args.get("_page", "1"))
+            if page < 1:
+                raise ValueError
         except ValueError:
-            return {"error": "page and page_size must be integers"}, 400
-        if page < 1:
-            return {"error": "page must be >= 1"}, 400
-        if page_size < 1:
-            return {"error": "page_size must be >= 1"}, 400
-        max_page_size = 200
-        if page_size > max_page_size:
-            page_size = max_page_size
+            return error_body("_page must be a positive integer", 400), 400
+        try:
+            page_size = parse_size_limit(
+                request.args.get("_size"), default=50, maximum=200
+            )
+        except ValueError as ex:
+            return error_body(str(ex), 400), 400
         offset = (page - 1) * page_size
 
         # Use the simplified allowed_resources method
@@ -306,6 +424,7 @@ class AllowedResourcesView(BaseView):
             # If catalog tables don't exist yet, return empty results
             return (
                 {
+                    "ok": True,
                     "action": action,
                     "actor_id": actor_id,
                     "page": page,
@@ -330,16 +449,17 @@ class AllowedResourcesView(BaseView):
         def build_page_url(page_number):
             pairs = []
             for key in request.args:
-                if key in {"page", "page_size"}:
+                if key in {"_page", "_size"}:
                     continue
                 for value in request.args.getlist(key):
                     pairs.append((key, value))
-            pairs.append(("page", str(page_number)))
-            pairs.append(("page_size", str(page_size)))
+            pairs.append(("_page", str(page_number)))
+            pairs.append(("_size", str(page_size)))
             query = urllib.parse.urlencode(pairs)
             return f"{request.path}?{query}"
 
         response = {
+            "ok": True,
             "action": action,
             "actor_id": actor_id,
             "page": page,
@@ -381,26 +501,24 @@ class PermissionRulesView(BaseView):
         # JSON API - action parameter is required
         action = request.args.get("action")
         if not action:
-            return Response.json({"error": "action parameter is required"}, status=400)
+            return Response.error("action parameter is required", 400)
         if action not in self.ds.actions:
-            return Response.json({"error": f"Unknown action: {action}"}, status=404)
+            return Response.error(f"Unknown action: {action}", 404)
 
         actor = request.actor if isinstance(request.actor, dict) else None
 
         try:
-            page = int(request.args.get("page", "1"))
-            page_size = int(request.args.get("page_size", "50"))
+            page = int(request.args.get("_page", "1"))
+            if page < 1:
+                raise ValueError
         except ValueError:
-            return Response.json(
-                {"error": "page and page_size must be integers"}, status=400
+            return Response.error("_page must be a positive integer", 400)
+        try:
+            page_size = parse_size_limit(
+                request.args.get("_size"), default=50, maximum=200
             )
-        if page < 1:
-            return Response.json({"error": "page must be >= 1"}, status=400)
-        if page_size < 1:
-            return Response.json({"error": "page_size must be >= 1"}, status=400)
-        max_page_size = 200
-        if page_size > max_page_size:
-            page_size = max_page_size
+        except ValueError as ex:
+            return Response.error(str(ex), 400)
         offset = (page - 1) * page_size
 
         from datasette.utils.actions_sql import build_permission_rules_sql
@@ -451,16 +569,17 @@ class PermissionRulesView(BaseView):
         def build_page_url(page_number):
             pairs = []
             for key in request.args:
-                if key in {"page", "page_size"}:
+                if key in {"_page", "_size"}:
                     continue
                 for value in request.args.getlist(key):
                     pairs.append((key, value))
-            pairs.append(("page", str(page_number)))
-            pairs.append(("page_size", str(page_size)))
+            pairs.append(("_page", str(page_number)))
+            pairs.append(("_size", str(page_size)))
             query = urllib.parse.urlencode(pairs)
             return f"{request.path}?{query}"
 
         response = {
+            "ok": True,
             "action": action,
             "actor_id": (actor or {}).get("id") if actor else None,
             "page": page,
@@ -481,17 +600,17 @@ class PermissionRulesView(BaseView):
 
 
 async def _check_permission_for_actor(ds, action, parent, child, actor):
-    """Shared logic for checking permissions. Returns a dict with check results."""
+    """Shared logic for checking and explaining a permission decision."""
     if action not in ds.actions:
-        return {"error": f"Unknown action: {action}"}, 404
+        return error_body(f"Unknown action: {action}", 404), 404
 
     if child and not parent:
-        return {"error": "parent is required when child is provided"}, 400
+        return error_body("parent is required when child is provided", 400), 400
 
     # Use the action's properties to create the appropriate resource object
     action_obj = ds.actions.get(action)
     if not action_obj:
-        return {"error": f"Unknown action: {action}"}, 400
+        return error_body(f"Unknown action: {action}", 400), 400
 
     # Global actions (no resource_class) don't have a resource
     if action_obj.resource_class is None:
@@ -506,18 +625,32 @@ async def _check_permission_for_actor(ds, action, parent, child, actor):
         resource_obj = action_obj.resource_class(parent)
     else:
         # This shouldn't happen given validation in Action.__post_init__
-        return {"error": f"Invalid action configuration: {action}"}, 500
+        return error_body(f"Invalid action configuration: {action}", 500), 500
 
     allowed = await ds.allowed(action=action, resource=resource_obj, actor=actor)
 
+    from datasette.utils.actions_sql import explain_permission_for_resource
+
+    explanation = await explain_permission_for_resource(
+        datasette=ds,
+        actor=actor,
+        action=action,
+        parent=parent,
+        child=child,
+    )
+
     response = {
+        "ok": True,
+        "unstable": UNSTABLE_API_MESSAGE,
         "action": action,
         "allowed": bool(allowed),
+        "actor": actor,
         "resource": {
             "parent": parent,
             "child": child,
             "path": _resource_path(parent, child),
         },
+        "explanation": explanation,
     }
 
     if actor and "id" in actor:
@@ -535,11 +668,25 @@ class PermissionCheckView(BaseView):
         as_format = request.url_vars.get("format")
 
         if not as_format:
+            actions = [
+                {
+                    "name": action.name,
+                    "description": action.description,
+                    "takes_parent": action.takes_parent,
+                    "takes_child": action.takes_child,
+                    "also_requires": action.also_requires,
+                }
+                for action in sorted(
+                    self.ds.actions.values(), key=lambda action: action.name
+                )
+            ]
             return await self.render(
                 ["debug_check.html"],
                 request,
                 {
-                    "sorted_actions": sorted(self.ds.actions.keys()),
+                    "actions": actions,
+                    "actor_json": request.args.get("actor")
+                    or json.dumps(request.actor, indent=2),
                     "has_debug_permission": True,
                 },
             )
@@ -547,13 +694,22 @@ class PermissionCheckView(BaseView):
         # JSON API - action parameter is required
         action = request.args.get("action")
         if not action:
-            return Response.json({"error": "action parameter is required"}, status=400)
+            return Response.error("action parameter is required", 400)
 
         parent = request.args.get("parent")
         child = request.args.get("child")
+        actor = request.actor
+        actor_json = request.args.get("actor")
+        if actor_json is not None:
+            try:
+                actor = json.loads(actor_json)
+            except json.JSONDecodeError as ex:
+                return Response.error(f"Invalid actor JSON: {ex}", 400)
+            if actor is not None and not isinstance(actor, dict):
+                return Response.error("actor must be a JSON object or null", 400)
 
         response, status = await _check_permission_for_actor(
-            self.ds, action, parent, child, request.actor
+            self.ds, action, parent, child, actor
         )
         return Response.json(response, status=status)
 
@@ -1094,7 +1250,7 @@ class JumpView(BaseView):
                 match["display_name"] = row["display_name"]
             matches.append(match)
 
-        return Response.json({"matches": matches, "truncated": truncated})
+        return Response.json({"ok": True, "matches": matches, "truncated": truncated})
 
 
 class SchemaBaseView(BaseView):
@@ -1116,7 +1272,7 @@ class SchemaBaseView(BaseView):
         headers = {}
         if self.ds.cors:
             add_cors_headers(headers)
-        return Response.json(data, headers=headers)
+        return Response.json({"ok": True, **data}, headers=headers)
 
     def format_error_response(self, error_message, format_, status=404):
         """Format error response based on requested format."""
@@ -1125,7 +1281,7 @@ class SchemaBaseView(BaseView):
             if self.ds.cors:
                 add_cors_headers(headers)
             return Response.json(
-                {"ok": False, "error": error_message}, status=status, headers=headers
+                error_body(error_message, status), status=status, headers=headers
             )
         else:
             return Response.text(error_message, status=status)
@@ -1201,17 +1357,17 @@ class DatabaseSchemaView(SchemaBaseView):
         database_name = request.url_vars["database"]
         format_ = request.url_vars.get("format") or "html"
 
-        # Check if database exists
-        if database_name not in self.ds.databases:
-            return self.format_error_response("Database not found", format_)
-
-        # Check view-database permission
+        # Permission check comes first, so actors without view-database
+        # cannot distinguish existing databases from missing ones
         await self.ds.ensure_permission(
             action="view-database",
             resource=DatabaseResource(database=database_name),
             actor=request.actor,
         )
 
+        if database_name not in self.ds.databases:
+            return self.format_error_response("Database not found", format_)
+
         schema = await self.get_database_schema(database_name)
 
         if format_ == "json":
@@ -1245,6 +1401,9 @@ class TableSchemaView(SchemaBaseView):
             actor=request.actor,
         )
 
+        if database_name not in self.ds.databases:
+            return self.format_error_response("Database not found", format_)
+
         # Get schema for the table
         db = self.ds.databases[database_name]
         result = await db.execute(

datasette/views/stored_queries.py

@@ -2,10 +2,10 @@ from urllib.parse import parse_qsl, urlencode
 
 from datasette.resources import DatabaseResource, QueryResource
 from datasette.stored_queries import stored_query_to_dict
-from datasette.utils import sqlite3, tilde_decode
+from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3, tilde_decode
 from datasette.utils.asgi import Response
 
-from .base import BaseView, _error
+from .base import BaseView
 from .query_helpers import (
     QueryValidationError,
     _as_bool,
@@ -34,12 +34,14 @@ class QueryParametersView(BaseView):
             resource=DatabaseResource(db.name),
             actor=request.actor,
         ):
-            return _block_framing(_error(["Permission denied: need execute-sql"], 403))
+            return _block_framing(
+                Response.error(["Permission denied: need execute-sql"], 403)
+            )
 
         invalid_keys = set(request.args) - {"sql"}
         if invalid_keys:
             return _block_framing(
-                _error(
+                Response.error(
                     ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))],
                     400,
                 )
@@ -47,8 +49,16 @@ class QueryParametersView(BaseView):
         try:
             parameters = _derived_query_parameters(request.args.get("sql") or "")
         except QueryValidationError as ex:
-            return _block_framing(_error([ex.message], ex.status))
-        return _block_framing(Response.json({"ok": True, "parameters": parameters}))
+            return _block_framing(Response.error([ex.message], ex.status))
+        return _block_framing(
+            Response.json(
+                {
+                    "ok": True,
+                    "unstable": UNSTABLE_API_MESSAGE,
+                    "parameters": parameters,
+                }
+            )
+        )
 
 
 def _query_list_url(path, query_string, *, set_args=None, remove_args=None):
@@ -82,11 +92,12 @@ class QueryListView(BaseView):
             limit = _query_list_limit(
                 request.args.get("_size"),
                 default=20 if format_ == "html" else 50,
+                maximum=self.ds.max_returned_rows,
             )
             is_write = _as_optional_bool(request.args.get("is_write"), "is_write")
             is_private = _as_optional_bool(request.args.get("is_private"), "is_private")
         except QueryValidationError as ex:
-            return _error([ex.message], ex.status)
+            return Response.error([ex.message], ex.status)
 
         page = await self.ds.list_queries(
             database,
@@ -111,9 +122,9 @@ class QueryListView(BaseView):
                 if key != "_next"
             ]
             pairs.append(("_next", page.next))
-            next_url = "{}?{}".format(
-                query_list_path,
-                urlencode(pairs),
+            next_url = self.ds.absolute_url(
+                request,
+                "{}?{}".format(request.path, urlencode(pairs)),
             )
 
         current_filters = {
@@ -199,7 +210,6 @@ class QueryListView(BaseView):
             "queries": page.queries,
             "next": page.next,
             "next_url": next_url,
-            "has_more": page.has_more,
             "limit": page.limit,
             "show_private_note": any(query.is_private for query in page.queries),
             "show_trusted_note": any(query.is_trusted for query in page.queries),
@@ -298,28 +308,30 @@ class QueryCreateAnalyzeView(BaseView):
             resource=DatabaseResource(db.name),
             actor=request.actor,
         ):
-            return _block_framing(_error(["Permission denied: need execute-sql"], 403))
+            return _block_framing(
+                Response.error(["Permission denied: need execute-sql"], 403)
+            )
         if not await self.ds.allowed(
             action="store-query",
             resource=DatabaseResource(db.name),
             actor=request.actor,
         ):
-            return _block_framing(_error(["Permission denied: need store-query"], 403))
+            return _block_framing(
+                Response.error(["Permission denied: need store-query"], 403)
+            )
 
         invalid_keys = set(request.args) - {"sql"}
         if invalid_keys:
             return _block_framing(
-                _error(
+                Response.error(
                     ["Invalid keys: {}".format(", ".join(sorted(invalid_keys)))],
                     400,
                 )
             )
         sql = request.args.get("sql") or ""
-        return _block_framing(
-            Response.json(
-                await _query_create_analysis_data(self.ds, db, sql, request.actor)
-            )
-        )
+        analysis = await _query_create_analysis_data(self.ds, db, sql, request.actor)
+        analysis["unstable"] = UNSTABLE_API_MESSAGE
+        return _block_framing(Response.json(analysis))
 
 
 class QueryStoreView(QueryCreateView):
@@ -346,13 +358,13 @@ class QueryStoreView(QueryCreateView):
             resource=DatabaseResource(db.name),
             actor=request.actor,
         ):
-            return _error(["Permission denied: need execute-sql"], 403)
+            return Response.error(["Permission denied: need execute-sql"], 403)
         if not await self.ds.allowed(
             action="store-query",
             resource=DatabaseResource(db.name),
             actor=request.actor,
         ):
-            return _error(["Permission denied: need store-query"], 403)
+            return Response.error(["Permission denied: need store-query"], 403)
 
         is_json = False
         query_data = {}
@@ -369,7 +381,7 @@ class QueryStoreView(QueryCreateView):
                 return await self._error_response(
                     request, db, query_data, ex.message, ex.status
                 )
-            return _error([ex.message], ex.status)
+            return Response.error([ex.message], ex.status)
 
         prepared.pop("analysis")
         name = prepared.pop("name")
@@ -378,13 +390,18 @@ class QueryStoreView(QueryCreateView):
         except sqlite3.IntegrityError as ex:
             if not is_json and isinstance(query_data, dict):
                 return await self._error_response(request, db, query_data, str(ex), 400)
-            return _error([str(ex)], 400)
+            return Response.error([str(ex)], 400)
 
         query = await self.ds.get_query(db.name, name)
         assert query is not None
         if is_json:
             return Response.json(
-                {"ok": True, "query": stored_query_to_dict(query)}, status=201
+                {
+                    "ok": True,
+                    "unstable": UNSTABLE_API_MESSAGE,
+                    "query": stored_query_to_dict(query),
+                },
+                status=201,
             )
         self.ds.add_message(request, "Query saved", self.ds.INFO)
         return Response.redirect(self.ds.urls.path(self.ds.urls.table(db.name, name)))
@@ -398,14 +415,20 @@ class QueryDefinitionView(BaseView):
         query_name = tilde_decode(request.url_vars["query"])
         query = await self.ds.get_query(db.name, query_name)
         if query is None:
-            return _error(["Query not found: {}".format(query_name)], 404)
+            return Response.error(["Query not found: {}".format(query_name)], 404)
         if not await self.ds.allowed(
             action="view-query",
             resource=QueryResource(db.name, query_name),
             actor=request.actor,
         ):
-            return _error(["Permission denied"], 403)
-        return Response.json({"ok": True, "query": stored_query_to_dict(query)})
+            return Response.error(["Permission denied"], 403)
+        return Response.json(
+            {
+                "ok": True,
+                "unstable": UNSTABLE_API_MESSAGE,
+                "query": stored_query_to_dict(query),
+            }
+        )
 
 
 class QueryUpdateView(BaseView):
@@ -416,15 +439,17 @@ class QueryUpdateView(BaseView):
         query_name = tilde_decode(request.url_vars["query"])
         existing = await self.ds.get_query(db.name, query_name)
         if existing is None:
-            return _error(["Query not found: {}".format(query_name)], 404)
+            return Response.error(["Query not found: {}".format(query_name)], 404)
         if not await self.ds.allowed(
             action="update-query",
             resource=QueryResource(db.name, query_name),
             actor=request.actor,
         ):
-            return _error(["Permission denied: need update-query"], 403)
+            return Response.error(["Permission denied: need update-query"], 403)
         if existing.is_trusted:
-            return _error(["Trusted queries cannot be updated using the API"], 403)
+            return Response.error(
+                ["Trusted queries cannot be updated using the API"], 403
+            )
 
         try:
             data, _ = await _json_or_form_payload(request)
@@ -450,7 +475,7 @@ class QueryUpdateView(BaseView):
                 self.ds, request, db, existing, update
             )
         except QueryValidationError as ex:
-            return _error([ex.message], ex.status)
+            return Response.error([ex.message], ex.status)
 
         await self.ds.update_query(db.name, query_name, **update_kwargs)
         if data.get("return"):
@@ -507,32 +532,32 @@ class QueryEditView(BaseView):
     async def get(self, request):
         db, query_name, existing = await self._load(request)
         if existing is None:
-            return _error(["Query not found: {}".format(query_name)], 404)
+            return Response.error(["Query not found: {}".format(query_name)], 404)
         await self.ds.ensure_permission(
             action="update-query",
             resource=QueryResource(db.name, query_name),
             actor=request.actor,
         )
         if existing.is_trusted:
-            return _error(["Trusted queries cannot be edited"], 403)
+            return Response.error(["Trusted queries cannot be edited"], 403)
         return await self._render_form(request, db, existing)
 
     async def post(self, request):
         db, query_name, existing = await self._load(request)
         if existing is None:
-            return _error(["Query not found: {}".format(query_name)], 404)
+            return Response.error(["Query not found: {}".format(query_name)], 404)
         if not await self.ds.allowed(
             action="update-query",
             resource=QueryResource(db.name, query_name),
             actor=request.actor,
         ):
-            return _error(["Permission denied: need update-query"], 403)
+            return Response.error(["Permission denied: need update-query"], 403)
         if existing.is_trusted:
-            return _error(["Trusted queries cannot be edited"], 403)
+            return Response.error(["Trusted queries cannot be edited"], 403)
 
         data, _ = await _json_or_form_payload(request)
         if not isinstance(data, dict):
-            return _error(["Invalid form submission"], 400)
+            return Response.error(["Invalid form submission"], 400)
         sql = data.get("sql")
         sql = existing.sql if sql is None else sql.strip()
         title = data.get("title") or ""
@@ -604,12 +629,16 @@ class QueryDeleteView(BaseView):
     async def get(self, request):
         db, query_name, existing = await self._load(request)
         if existing is None:
-            return _error(["Query not found: {}".format(query_name)], 404)
+            return Response.error(["Query not found: {}".format(query_name)], 404)
         await self.ds.ensure_permission(
             action="delete-query",
             resource=QueryResource(db.name, query_name),
             actor=request.actor,
         )
+        if existing.is_trusted:
+            return Response.error(
+                ["Trusted queries cannot be deleted using the API"], 403
+            )
         return await self.render(
             ["query_delete.html"],
             request,
@@ -624,13 +653,17 @@ class QueryDeleteView(BaseView):
     async def post(self, request):
         db, query_name, existing = await self._load(request)
         if existing is None:
-            return _error(["Query not found: {}".format(query_name)], 404)
+            return Response.error(["Query not found: {}".format(query_name)], 404)
         if not await self.ds.allowed(
             action="delete-query",
             resource=QueryResource(db.name, query_name),
             actor=request.actor,
         ):
-            return _error(["Permission denied: need delete-query"], 403)
+            return Response.error(["Permission denied: need delete-query"], 403)
+        if existing.is_trusted:
+            return Response.error(
+                ["Trusted queries cannot be deleted using the API"], 403
+            )
 
         data, is_json = await _json_or_form_payload(request)
         await self.ds.remove_query(db.name, query_name)

datasette/views/table_extras.py

@@ -1,10 +1,11 @@
 import itertools
 from dataclasses import dataclass
 
+from datasette.column_types import SQLiteType
 from datasette.database import QueryInterrupted
 from datasette.extras import Extra, ExtraExample, ExtraRegistry, ExtraScope, Provider
 from datasette.plugins import pm
-from datasette.resources import TableResource
+from datasette.resources import DatabaseResource, TableResource
 from datasette.utils import (
     await_me_maybe,
     call_with_supported_arguments,
@@ -98,7 +99,7 @@ class QueryExtraContext:
 
 
 class CountSqlExtra(Extra):
-    description = "SQL query used to calculate the total count"
+    description = "SQL query string used to calculate the total count for the current table view, including active filters."
     example = ExtraExample("/fixtures/facetable.json?_size=0&_extra=count_sql")
     scopes = {ExtraScope.TABLE}
 
@@ -127,8 +128,8 @@ class CountExtra(Extra):
                 pass
 
         if context.count_sql and count is None and not context.nocount:
-            count_sql_limited = (
-                f"select count(*) from (select * {context.from_sql} limit 10001)"
+            count_sql_limited = "select count(*) from (select * {} limit {})".format(
+                context.from_sql, context.db.count_limit + 1
             )
             try:
                 count_rows = list(
@@ -140,6 +141,39 @@ class CountExtra(Extra):
         return count
 
 
+def count_is_truncated(datasette, db, database_name, table_name, count_sql, count):
+    if count != db.count_limit + 1:
+        return False
+    if (
+        not db.is_mutable
+        and datasette.inspect_data
+        and count_sql == f"select count(*) from {table_name} "
+    ):
+        try:
+            datasette.inspect_data[database_name]["tables"][table_name]["count"]
+            return False
+        except KeyError:
+            pass
+    return True
+
+
+class CountTruncatedExtra(Extra):
+    description = "True if the count hit Datasette's counting limit, meaning the real number of matching rows is at least the reported count."
+    example = ExtraExample("/fixtures/facetable.json?_extra=count,count_truncated")
+    scopes = {ExtraScope.TABLE}
+    expensive = True
+
+    async def resolve(self, context, count):
+        return count_is_truncated(
+            context.datasette,
+            context.db,
+            context.database_name,
+            context.table_name,
+            context.count_sql,
+            count,
+        )
+
+
 class FacetInstancesProvider(Provider):
     scopes = {ExtraScope.TABLE}
 
@@ -165,7 +199,7 @@ class FacetInstancesProvider(Provider):
 
 
 class FacetResultsExtra(Extra):
-    description = "Results of facets calculated against this data"
+    description = "Results of facets calculated against this data. A dictionary with ``results`` and ``timed_out`` keys: ``results`` maps facet names to facet dictionaries with ``name``, ``type``, ``results`` and URL keys, and each facet result item includes ``value``, ``label``, ``count`` and ``toggle_url``."
     example = ExtraExample(
         value={
             "results": {
@@ -184,6 +218,7 @@ class FacetResultsExtra(Extra):
     )
     scopes = {ExtraScope.TABLE}
     expensive = True
+    docs_note = "See :ref:`facets` for details of how facets work."
 
     async def resolve(self, context, facet_instances):
         facet_results = {}
@@ -213,9 +248,16 @@ class FacetResultsExtra(Extra):
 
 
 class FacetsTimedOutExtra(Extra):
-    description = "Facet calculations that timed out"
+    description = (
+        "List of names of facet calculations that exceeded the facet time limit."
+    )
     example = ExtraExample(
-        "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out"
+        "/fixtures/facetable.json?_facet=state&_extra=facets_timed_out",
+        note=(
+            "A list of the names of any facets that exceeded the "
+            ":ref:`setting_facet_time_limit_ms` time limit - an empty list "
+            "if every facet calculation completed."
+        ),
     )
     scopes = {ExtraScope.TABLE}
 
@@ -224,7 +266,7 @@ class FacetsTimedOutExtra(Extra):
 
 
 class SuggestedFacetsExtra(Extra):
-    description = "Suggestions for facets that might return interesting results"
+    description = "Suggestions for facets that might return interesting results. Each item is a dictionary with ``name`` and ``toggle_url`` keys, and may include extra keys such as ``type`` or ``label`` depending on the facet class."
     example = ExtraExample(
         value=[
             {
@@ -236,6 +278,9 @@ class SuggestedFacetsExtra(Extra):
     )
     scopes = {ExtraScope.TABLE}
     expensive = True
+    docs_note = (
+        "Suggestions are controlled by the :ref:`setting_suggest_facets` setting."
+    )
 
     async def resolve(self, context, facet_instances):
         suggested_facets = []
@@ -276,17 +321,8 @@ class HumanDescriptionEnExtra(Extra):
         return human_description_en
 
 
-class NextUrlExtra(Extra):
-    description = "Full URL for the next page of results"
-    example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=next_url")
-    scopes = {ExtraScope.TABLE}
-
-    async def resolve(self, context):
-        return context.next_url
-
-
 class ColumnsExtra(Extra):
-    description = "Column names returned by this query"
+    description = "List of column names returned by this table, row or query."
     example = ExtraExample("/fixtures/facetable.json?_extra=columns")
     examples = {
         ExtraScope.ROW: ExtraExample(
@@ -303,7 +339,7 @@ class ColumnsExtra(Extra):
 
 
 class AllColumnsExtra(Extra):
-    description = "All columns in the table, regardless of _col/_nocol filtering"
+    description = "List of all column names in the table, regardless of ``_col=`` or ``_nocol=`` filtering."
     example = ExtraExample("/fixtures/facetable.json?_col=pk&_extra=all_columns")
     scopes = {ExtraScope.TABLE}
 
@@ -312,7 +348,7 @@ class AllColumnsExtra(Extra):
 
 
 class PrimaryKeysExtra(Extra):
-    description = "Primary keys for this table"
+    description = "List of primary key column names for this table, or an empty list if the table has no explicit primary key."
     example = ExtraExample("/fixtures/facetable.json?_extra=primary_keys")
     examples = {
         ExtraScope.ROW: ExtraExample(
@@ -325,8 +361,57 @@ class PrimaryKeysExtra(Extra):
         return context.pks
 
 
+def column_detail_as_json(column):
+    return {
+        "type": column.type,
+        "sqlite_type": SQLiteType.from_declared_type(column.type).value,
+        "notnull": bool(column.notnull),
+        "default": column.default_value,
+        "is_pk": bool(column.is_pk),
+        "pk_position": column.is_pk,
+        "hidden": column.hidden,
+    }
+
+
+class ColumnDetailsExtra(Extra):
+    description = (
+        "SQLite schema details for columns in this table. The dictionary maps "
+        "column names to objects describing the schema for each column."
+    )
+    docs_note = (
+        "Each object has ``type`` as the declared type string returned by "
+        'SQLite, or ``""`` if no type was declared; ``sqlite_type`` as the '
+        "normalized SQLite affinity, one of ``TEXT``, ``INTEGER``, ``REAL``, "
+        "``BLOB`` or ``NUMERIC``; ``notnull`` as a boolean; ``default`` "
+        'as the raw SQL default expression string, such as ``"42"``, '
+        "``\"'hello'\"`` or ``\"datetime('now')\"``, or ``null`` if there is "
+        "no default; ``is_pk`` as a boolean; ``pk_position`` as the integer "
+        "primary key position reported by SQLite, or ``0`` for columns that "
+        "are not part of the primary key; and ``hidden`` as the integer value "
+        "reported by SQLite's ``PRAGMA table_xinfo``. ``hidden`` is ``0`` for "
+        "normal columns, ``1`` for hidden virtual table columns, ``2`` for "
+        "virtual generated columns and ``3`` for stored generated columns."
+    )
+    example = ExtraExample("/fixtures/binary_data.json?_size=0&_extra=column_details")
+    examples = {
+        ExtraScope.ROW: ExtraExample(
+            "/fixtures/binary_data/1.json?_extra=column_details"
+        )
+    }
+    scopes = {ExtraScope.TABLE, ExtraScope.ROW}
+
+    async def resolve(self, context):
+        column_details = await context.datasette._get_resource_column_details(
+            context.database_name, context.table_name
+        )
+        return {
+            column_name: column_detail_as_json(column)
+            for column_name, column in column_details.items()
+        }
+
+
 class ActionsExtra(Extra):
-    description = "Table or view actions made available by plugin hooks"
+    description = 'Async callable returning table or view actions made available by core and plugin hooks. Each item is either a link with ``href``, ``label`` and optional ``description`` keys, or a button with ``type: "button"``, ``label``, optional ``description`` and optional ``attrs``. See :ref:`plugin_actions`, :ref:`plugin_hook_table_actions` and :ref:`plugin_hook_view_actions`.'
     scopes = {ExtraScope.TABLE}
     # Returns an async function for the HTML templates - not JSON serializable
     public = False
@@ -346,6 +431,21 @@ class ActionsExtra(Extra):
             else:
                 kwargs["table"] = context.table_name
                 method = pm.hook.table_actions
+                # Resolve the registered table-level actions for this table
+                # and the database-level actions for its database in two
+                # batched queries, seeding the request permission cache so
+                # that allowed() calls made inside the plugin hooks below
+                # are served from the cache
+                datasette = context.datasette
+                await precompute_table_action_permissions(
+                    datasette,
+                    context.request.actor,
+                    context.database_name,
+                    context.table_name,
+                )
+                await precompute_database_action_permissions(
+                    datasette, context.request.actor, context.database_name
+                )
             for hook in method(**kwargs):
                 extra_links = await await_me_maybe(hook)
                 if extra_links:
@@ -355,6 +455,32 @@ class ActionsExtra(Extra):
         return actions
 
 
+async def precompute_table_action_permissions(
+    datasette, actor, database_name, table_name
+):
+    await datasette.allowed_many(
+        actions=[
+            name
+            for name, action in datasette.actions.items()
+            if action.resource_class is TableResource
+        ],
+        resource=TableResource(database_name, table_name),
+        actor=actor,
+    )
+
+
+async def precompute_database_action_permissions(datasette, actor, database_name):
+    await datasette.allowed_many(
+        actions=[
+            name
+            for name, action in datasette.actions.items()
+            if action.resource_class is DatabaseResource
+        ],
+        resource=DatabaseResource(database_name),
+        actor=actor,
+    )
+
+
 class IsViewExtra(Extra):
     description = "Whether this resource is a view instead of a table"
     example = ExtraExample("/fixtures/simple_view.json?_extra=is_view")
@@ -365,7 +491,11 @@ class IsViewExtra(Extra):
 
 
 class DebugExtra(Extra):
-    description = "Extra debug information"
+    description = "Extra debug information dictionary. This is intended for development only and its shape is not part of the stable template contract."
+    docs_note = (
+        "The contents of this block are not a stable part of the Datasette "
+        "API and may change without warning."
+    )
     example = ExtraExample("/fixtures/facetable.json?_extra=debug")
     examples = {
         ExtraScope.ROW: ExtraExample(
@@ -397,7 +527,7 @@ class DebugExtra(Extra):
 
 
 class RequestExtra(Extra):
-    description = "Full information about the request"
+    description = "Dictionary with request details: ``url``, ``path``, ``full_path``, ``host`` and ``args`` where ``args`` maps query string parameter names to their values."
     example = ExtraExample("/fixtures/facetable.json?_extra=request")
     examples = {
         ExtraScope.ROW: ExtraExample(
@@ -441,7 +571,7 @@ class DisplayColumnsAndRowsProvider(Provider):
 
 
 class DisplayColumnsExtra(Extra):
-    description = "Column metadata used by the HTML table display"
+    description = "Column metadata used by the HTML table display. Each item includes ``name``, ``sortable``, ``is_pk``, ``type``, ``notnull``, ``description``, ``column_type`` and ``column_type_config`` keys."
     example = ExtraExample(
         value=[
             {
@@ -471,7 +601,7 @@ class DisplayColumnsExtra(Extra):
 
 
 class DisplayRowsExtra(Extra):
-    description = "Row data formatted for the HTML table display"
+    description = "Rows formatted for the HTML table display. Each row is iterable and contains cell dictionaries with ``column``, ``value``, ``raw`` and ``value_type`` keys; table pages may also provide ``pk_path``, ``row_path`` and ``row_label`` attributes on each row object."
     scopes = {ExtraScope.TABLE}
     # Contains markupsafe/sqlite3.Row values - not JSON serializable
     public = False
@@ -482,6 +612,10 @@ class DisplayRowsExtra(Extra):
 
 class RenderCellExtra(Extra):
     description = "Rendered HTML for each cell using the render_cell plugin hook"
+    docs_note = (
+        "See the :ref:`render_cell() plugin hook <plugin_hook_render_cell>` "
+        "documentation."
+    )
     example = ExtraExample(
         value={
             "rows": [
@@ -576,7 +710,7 @@ class RenderCellExtra(Extra):
 
 
 class QueryExtra(Extra):
-    description = "Details of the underlying SQL query"
+    description = "Details of the underlying SQL query as a dictionary with ``sql`` and ``params`` keys."
     example = ExtraExample("/fixtures/facetable.json?_size=1&_extra=query")
     examples = {
         ExtraScope.ROW: ExtraExample(
@@ -597,8 +731,29 @@ class QueryExtra(Extra):
 
 
 class ColumnTypesExtra(Extra):
-    description = "Column type assignments for this table"
-    example = ExtraExample(value={})
+    description = 'Column type assignments for this table. A dictionary mapping column names to ``{"type": type_name, "config": config}`` dictionaries.'
+    docs_note = (
+        "An empty object if no column types have been assigned. Column types "
+        "can be assigned in :ref:`configuration "
+        "<table_configuration_column_types>` or using the :ref:`set column "
+        "type API <TableSetColumnTypeView>`."
+    )
+    example = ExtraExample(
+        "/fixtures/facetable.json?_size=0&_extra=column_types",
+        note=(
+            "This example is from an instance where the ``tags`` column has "
+            "been assigned the ``json`` column type."
+        ),
+    )
+    examples = {
+        ExtraScope.ROW: ExtraExample(
+            "/fixtures/facetable/1.json?_extra=column_types",
+            note=(
+                "This example is from an instance where the ``tags`` column "
+                "has been assigned the ``json`` column type."
+            ),
+        )
+    }
     scopes = {ExtraScope.TABLE, ExtraScope.ROW}
 
     async def resolve(self, context):
@@ -615,7 +770,40 @@ class ColumnTypesExtra(Extra):
 
 
 class SetColumnTypeUiExtra(Extra):
-    description = "Column type UI metadata for this table"
+    description = "Information needed to build an interface for assigning column types, or ``None`` if unavailable. When present it has ``path`` and ``columns`` keys; ``columns`` maps column names to ``current`` and ``options`` values."
+    docs_note = (
+        "``null`` unless the current actor is allowed to use the :ref:`set "
+        "column type API <TableSetColumnTypeView>` for this table."
+    )
+    example = ExtraExample(
+        value={
+            "path": "/fixtures/facetable/-/set-column-type",
+            "columns": {
+                "created": {
+                    "current": None,
+                    "options": [
+                        {"name": "email", "description": "Email address"},
+                        {"name": "json", "description": "JSON data"},
+                        {"name": "url", "description": "URL"},
+                    ],
+                },
+                "tags": {
+                    "current": {"type": "json", "config": None},
+                    "options": [
+                        {"name": "email", "description": "Email address"},
+                        {"name": "json", "description": "JSON data"},
+                        {"name": "url", "description": "URL"},
+                    ],
+                },
+            },
+        },
+        note=(
+            "Shape abbreviated to two columns, as seen by an actor with "
+            "``set-column-type`` permission. ``current`` is the column type "
+            "currently assigned to each column and ``options`` lists the "
+            "types that could be assigned to it."
+        ),
+    )
     scopes = {ExtraScope.TABLE}
 
     async def resolve(self, context):
@@ -666,14 +854,34 @@ class SetColumnTypeUiExtra(Extra):
 
 
 class MetadataExtra(Extra):
-    description = "Metadata about the table, database or stored query"
-    example = ExtraExample("/fixtures/facetable.json?_extra=metadata")
+    description = "Metadata dictionary for the table, database or stored query. Table and row metadata include a ``columns`` dictionary mapping column names to descriptions; stored query metadata returns the stored query configuration."
+    docs_note = "See :ref:`metadata` for how to attach metadata to tables."
+    example = ExtraExample(
+        "/fixtures/facetable.json?_extra=metadata",
+        note=(
+            "This example is from an instance where the ``facetable`` table "
+            "has a metadata ``description`` and a :ref:`column description "
+            "<metadata_column_descriptions>` for its ``state`` column. The "
+            "``columns`` object is empty for tables with no column "
+            "descriptions."
+        ),
+    )
     examples = {
         ExtraScope.ROW: ExtraExample(
-            "/fixtures/simple_primary_key/1.json?_extra=metadata"
+            "/fixtures/simple_primary_key/1.json?_extra=metadata",
+            note=(
+                "This table has no metadata, so only an empty ``columns`` "
+                "object is returned."
+            ),
         ),
         ExtraScope.QUERY: ExtraExample(
-            "/fixtures/neighborhood_search.json?text=town&_extra=metadata"
+            "/fixtures/neighborhood_search.json?text=town&_extra=metadata",
+            note=(
+                "For stored queries this returns the full configuration of "
+                "the query, including the :ref:`stored query options "
+                "<queries_options>`. For ``?sql=`` queries it returns an "
+                "empty object."
+            ),
         ),
     }
     scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}
@@ -733,6 +941,10 @@ class TableExtra(Extra):
 
 class DatabaseColorExtra(Extra):
     description = "Color assigned to the database"
+    docs_note = (
+        "A six character hex color, without the leading ``#``, derived from "
+        "a hash of the database name and used in the Datasette interface."
+    )
     example = ExtraExample("/fixtures/facetable.json?_extra=database_color")
     examples = {
         ExtraScope.ROW: ExtraExample(
@@ -749,7 +961,7 @@ class DatabaseColorExtra(Extra):
 
 
 class FormHiddenArgsExtra(Extra):
-    description = "Hidden form arguments used by the HTML table interface"
+    description = "List of ``(name, value)`` pairs for hidden form fields used by the HTML table interface to preserve current query string options."
     example = ExtraExample(
         "/fixtures/facetable.json?_facet=state&_size=1&_extra=form_hidden_args"
     )
@@ -769,7 +981,7 @@ class FormHiddenArgsExtra(Extra):
 
 
 class FiltersExtra(Extra):
-    description = "Filters object used by the HTML table interface"
+    description = "``Filters`` object used by the HTML table interface. Useful methods include ``filters.human_description_en()``; this is not JSON serializable."
     scopes = {ExtraScope.TABLE}
     # Returns a Filters instance for the HTML templates - not JSON serializable
     public = False
@@ -779,7 +991,12 @@ class FiltersExtra(Extra):
 
 
 class CustomTableTemplatesExtra(Extra):
-    description = "Custom template names considered for this table"
+    description = "List of custom template names considered for rendering table rows, in lookup order."
+    docs_note = (
+        "The first template in this list that exists will be used to render "
+        "the table on the HTML version of this page. See "
+        ":ref:`customization_custom_templates`."
+    )
     example = ExtraExample("/fixtures/facetable.json?_extra=custom_table_templates")
     scopes = {ExtraScope.TABLE}
 
@@ -792,7 +1009,13 @@ class CustomTableTemplatesExtra(Extra):
 
 
 class SortedFacetResultsExtra(Extra):
-    description = "Facet results sorted for display"
+    description = "Facet result dictionaries sorted for display. Each item has the same shape as an entry from ``facet_results['results']``."
+    docs_note = (
+        "The same data as ``facet_results``, as a list in the order used by "
+        "the HTML interface: facets from :ref:`facet configuration "
+        "<facets_metadata>` first, then other facets ordered by their number "
+        "of results."
+    )
     example = ExtraExample(
         "/fixtures/facetable.json?_facet=state&_extra=sorted_facet_results"
     )
@@ -848,8 +1071,16 @@ class ViewDefinitionExtra(Extra):
 
 
 class RenderersExtra(Extra):
-    description = "Alternative output renderers available for this table"
-    example = ExtraExample("/fixtures/facetable.json?_extra=renderers")
+    description = "Dictionary mapping output format names such as ``json`` or plugin-provided renderer names to URLs for this data in that format."
+    example = ExtraExample(
+        "/fixtures/facetable.json?_extra=renderers",
+        note=(
+            "Each key is the name of an output format, each value the URL "
+            "for this data in that format. Plugins can add additional "
+            "formats using the :ref:`register_output_renderer() plugin hook "
+            "<plugin_register_output_renderer>`."
+        ),
+    )
     scopes = {ExtraScope.TABLE}
 
     async def resolve(self, context, expandable_columns, query):
@@ -887,6 +1118,10 @@ class RenderersExtra(Extra):
 
 class PrivateExtra(Extra):
     description = "Whether this resource is private to the current actor"
+    docs_note = (
+        "``true`` if the current actor can see this resource but an "
+        "anonymous user could not. See :ref:`authentication_permissions`."
+    )
     example = ExtraExample("/fixtures/facetable.json?_extra=private")
     examples = {
         ExtraScope.ROW: ExtraExample(
@@ -903,8 +1138,16 @@ class PrivateExtra(Extra):
 
 
 class ExpandableColumnsExtra(Extra):
-    description = "Foreign key columns that can be expanded with labels"
-    example = ExtraExample("/fixtures/facetable.json?_extra=expandable_columns")
+    description = "List of foreign key columns that can be expanded with labels. Each item is a ``(foreign_key, label_column)`` pair where ``foreign_key`` is the SQLite foreign key dictionary and ``label_column`` is the label column in the referenced table, or ``None``."
+    docs_note = "See :ref:`expand_foreign_keys` for how to expand these labels."
+    example = ExtraExample(
+        "/fixtures/facetable.json?_extra=expandable_columns",
+        note=(
+            "Each item is a ``[foreign_key, label_column]`` pair: the "
+            "foreign key relationship, then the column in the other table "
+            "that would be used as the label for each expanded value."
+        ),
+    )
     scopes = {ExtraScope.TABLE}
 
     async def resolve(self, context):
@@ -917,11 +1160,16 @@ class ExpandableColumnsExtra(Extra):
 
 
 class ForeignKeyTablesExtra(Extra):
-    description = "Tables that link to this row using foreign keys"
+    description = "List of tables that link to this row using foreign keys. Each item includes the foreign key fields plus ``count`` for matching rows and ``link`` for the filtered table URL."
     example = ExtraExample(
-        "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables"
+        "/fixtures/simple_primary_key/1.json?_extra=foreign_key_tables",
+        note=(
+            "``count`` is the number of rows in the other table that "
+            "reference this row, and ``link`` is a URL to browse those rows."
+        ),
     )
     scopes = {ExtraScope.ROW}
+    expensive = True
 
     async def resolve(self, context):
         return await context.foreign_key_tables(
@@ -930,7 +1178,30 @@ class ForeignKeyTablesExtra(Extra):
 
 
 class ExtrasExtra(Extra):
-    description = "Available ?_extra= blocks"
+    description = "List of ``?_extra=`` blocks that can be used on this page. Each item has ``name``, ``description``, ``toggle_url`` and ``selected`` keys."
+    example = ExtraExample(
+        value=[
+            {
+                "name": "count",
+                "description": "Total count of rows matching these filters",
+                "toggle_url": "http://localhost/fixtures/facetable.json?_extra=extras&_extra=count",
+                "selected": False,
+            },
+            {
+                "name": "extras",
+                "description": "List of ?_extra= blocks that can be used on this page",
+                "toggle_url": "http://localhost/fixtures/facetable.json",
+                "selected": True,
+            },
+        ],
+        note=(
+            "Shape abbreviated from /fixtures/facetable.json?_extra=extras - "
+            "the full response lists every extra described on this page. "
+            "``toggle_url`` is the current URL with that extra added or "
+            "removed, and ``selected`` is ``true`` for extras included in "
+            "the current request."
+        ),
+    )
     scopes = {ExtraScope.TABLE, ExtraScope.ROW, ExtraScope.QUERY}
 
     async def resolve(self, context):
@@ -964,7 +1235,6 @@ TABLE_EXTRA_BUNDLES = {
         "count",
         "count_sql",
         "human_description_en",
-        "next_url",
         "metadata",
         "query",
         "columns",
@@ -993,16 +1263,17 @@ TABLE_EXTRA_BUNDLES = {
 
 TABLE_EXTRA_CLASSES = [
     CountExtra,
+    CountTruncatedExtra,
     CountSqlExtra,
     FacetResultsExtra,
     FacetsTimedOutExtra,
     SuggestedFacetsExtra,
     FacetInstancesProvider,
     HumanDescriptionEnExtra,
-    NextUrlExtra,
     ColumnsExtra,
     AllColumnsExtra,
     PrimaryKeysExtra,
+    ColumnDetailsExtra,
     DisplayColumnsAndRowsProvider,
     DisplayColumnsExtra,
     DisplayRowsExtra,

datasette/write_sql.py

@@ -138,6 +138,33 @@ def decision_for_write_sql_operation(
                 ),
             )
         )
+    if operation.operation == "create" and operation.target_type == "view":
+        if operation.database is None:
+            return UnsupportedWriteSqlOperation(unsupported_message)
+        return RequireWriteSqlPermissions(
+            (
+                PermissionRequirement(
+                    action="create-view",
+                    resource=DatabaseResource(database=operation.database),
+                ),
+            )
+        )
+    if (
+        operation.operation == "drop"
+        and operation.target_type == "view"
+        and operation.database is not None
+        and operation.table is not None
+    ):
+        return RequireWriteSqlPermissions(
+            (
+                PermissionRequirement(
+                    action="drop-view",
+                    resource=TableResource(
+                        database=operation.database, table=operation.table
+                    ),
+                ),
+            )
+        )
     if (
         operation.operation == "alter"
         and operation.target_type == "table"

docs/Makefile

@@ -20,4 +20,4 @@ help:
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 livehtml:
-	sphinx-autobuild -b html "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0)
+	sphinx-autobuild -b html --watch ../datasette "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(0)

docs/authentication.rst

@@ -21,6 +21,23 @@ The actor dictionary can be any shape - the design of that data structure is lef
 
 Plugins can use the :ref:`plugin_hook_actor_from_request` hook to implement custom logic for authenticating an actor based on the incoming HTTP request.
 
+.. _authentication_actor_display:
+
+How actors are displayed
+------------------------
+
+In a number of places - such as the navigation menu and the ``/-/logout`` page - Datasette needs to display a short label representing the currently authenticated actor.
+
+To decide what to show, Datasette looks through the following keys in the actor dictionary and uses the value of the first one that is present and not empty:
+
+* ``display``
+* ``name``
+* ``username``
+* ``login``
+* ``id``
+
+If none of those keys have a value the actor dictionary is displayed as a string instead.
+
 .. _authentication_root:
 
 Using the "root" actor
@@ -28,12 +45,12 @@ Using the "root" actor
 
 Datasette currently leaves almost all forms of authentication to plugins - `datasette-auth-github <https://github.com/simonw/datasette-auth-github>`__ for example.
 
-The one exception is the "root" account, which you can sign into while using Datasette on your local machine. The root user has **all permissions** - they can perform any action regardless of other permission rules.
+The one exception is the "root" account, which you can sign into while using Datasette on your local machine. The root user starts with **all permissions**: Datasette contributes a global allow rule for every action. More specific deny rules can still override that global rule.
 
 The ``--root`` flag is designed for local development and testing. When you start Datasette with ``--root``, the root user automatically receives every permission, including:
 
 * All view permissions (``view-instance``, ``view-database``, ``view-table``, etc.)
-* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``alter-table``, ``set-column-type``, ``drop-table``)
+* All write permissions (``insert-row``, ``update-row``, ``delete-row``, ``create-table``, ``create-view``, ``alter-table``, ``set-column-type``, ``drop-table``, ``drop-view``)
 * Debug permissions (``permissions-debug``, ``debug-menu``)
 * Any custom permissions defined by plugins
 
@@ -67,12 +84,12 @@ Click on that link and then visit ``http://127.0.0.1:8001/-/actor`` to confirm t
 Permissions
 ===========
 
-Datasette's permissions system is built around SQL queries. Datasette and its plugins construct SQL queries to resolve the list of resources that an actor cas access.
-
 The key question the permissions system answers is this:
 
     Is this **actor** allowed to perform this **action**, optionally against this particular **resource**?
 
+Every permission decision can be understood in terms of those three values. Datasette implements the decisions using SQL, but you do not need to understand the generated SQL to configure or debug permissions.
+
 **Actors** are :ref:`described above <authentication_actor>`.
 
 An **action** is a string describing the action the actor would like to perform. A full list is :ref:`provided below <actions>` - examples include ``view-table`` and ``execute-sql``.
@@ -121,7 +138,51 @@ This configuration will deny access to everyone except the user with ``id`` of `
 How permissions are resolved
 ----------------------------
 
-Datasette performs permission checks using the internal :ref:`datasette_allowed`, method which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``.
+Permission rules describe an effect (``allow`` or ``deny``) at one of three levels:
+
+``resource``
+    A specific child resource, such as the ``analytics/sales`` table.
+
+``parent``
+    A parent resource, such as the ``analytics`` database. A parent rule also applies to its child resources.
+
+``global``
+    Every resource for that action.
+
+Datasette resolves matching rules from most specific to least specific:
+
+#. Resource rules take precedence over parent and global rules.
+#. Parent rules take precedence over global rules.
+#. If both allow and deny rules match at the same level, deny takes precedence.
+#. If no rule matches, access is denied.
+
+This means a resource-level allow can provide an exception to a parent-level deny. It also means that two plugins which disagree at the same level resolve to deny.
+
+.. list-table:: Permission rule examples
+   :header-rows: 1
+
+   * - Matching rules
+     - Result
+     - Explanation
+   * - Global allow
+     - Allow
+     - The global rule is the most specific matching rule.
+   * - Global allow, parent deny
+     - Deny
+     - The parent rule is more specific.
+   * - Parent deny, resource allow
+     - Allow
+     - The resource rule is more specific.
+   * - Resource allow and resource deny
+     - Deny
+     - Deny takes precedence at the same level.
+   * - No matching rules
+     - Deny
+     - Permissions default to deny when no rule applies.
+
+The built-in public defaults are global allow rules for actions such as ``view-instance``, ``view-database`` and ``view-table``. They follow the same precedence rules as configuration and plugin rules. The ``--default-deny`` option prevents Datasette from contributing those default allow rules.
+
+Datasette performs checks using :ref:`datasette_allowed`, which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``.
 
 ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified.
 
@@ -132,12 +193,12 @@ resources were allowed or denied. The combined sources are:
 
 * ``allow`` blocks configured in :ref:`datasette.yaml <authentication_permissions_config>`.
 * :ref:`Actor restrictions <authentication_cli_create_token_restrict>` encoded into the actor dictionary or API token.
-* The "root" user shortcut when ``--root`` (or :attr:`Datasette.root_enabled <datasette.app.Datasette.root_enabled>`) is active, replying ``True`` to all permission chucks unless configuration rules deny them at a more specific level.
+* The "root" user rule when ``--root`` (or :attr:`Datasette.root_enabled <datasette.app.Datasette.root_enabled>`) is active. This is a global allow rule, so a more specific configuration deny can override it.
 * Any additional SQL provided by plugins implementing :ref:`plugin_hook_permission_resources_sql`.
 
-Datasette evaluates the SQL to determine if the requested ``resource`` is
-included. Explicit deny rules returned by configuration or plugins will block
-access even if other rules allowed it.
+Actor restrictions are applied after the allow/deny rules. They act as an additional allowlist: a restriction can remove access but cannot grant access that the actor did not already have. See :ref:`authentication_cli_create_token_restrict`.
+
+Some actions have dependencies on other actions. These are evaluated as an ``AND`` condition. For example, ``execute-sql`` also requires ``view-database``: both decisions must be allowed for the final result to be allowed.
 
 .. _authentication_permissions_allow:
 
@@ -974,7 +1035,9 @@ The ``/-/create-token`` page cannot be accessed by actors that are authenticated
 
 Datasette plugins that implement their own form of API token authentication should follow this convention.
 
-You can disable the signed token feature entirely using the :ref:`allow_signed_tokens <setting_allow_signed_tokens>` setting.
+If a request presents a token that a token handler recognizes but rejects - an invalid signature, a malformed payload or an expired token - Datasette responds with a ``401`` status, the :ref:`standard JSON error format <json_api_errors>` and a ``WWW-Authenticate: Bearer error="invalid_token"`` header. This means API clients can distinguish "your token needs to be renewed" (``401``) from "your token does not grant this permission" (``403``). A ``Bearer`` token that no registered handler recognizes at all is ignored, since it may be intended for an authentication plugin.
+
+You can disable the signed token feature entirely using the :ref:`allow_signed_tokens <setting_allow_signed_tokens>` setting. Requests presenting a ``dstok_`` token while the feature is disabled receive a ``401``.
 
 .. _authentication_cli_create_token:
 
@@ -1126,11 +1189,23 @@ The debug tool at ``/-/permissions`` is available to any actor with the ``permis
 
     datasette -s permissions.permissions-debug true data.db
 
-The page shows the permission checks that have been carried out by the Datasette instance.
+The permission debug tools answer four different questions:
 
-It also provides an interface for running hypothetical permission checks against a hypothetical actor. This is a useful way of confirming that your configured permissions work in the way you expect.
+Why was this decision allowed or denied?
+    Use :ref:`PermissionCheckView`. It shows every matching rule, identifies the winning specificity level, applies actor restrictions and evaluates any required actions.
 
-This is designed to help administrators and plugin authors understand exactly how permission checks are being carried out, in order to effectively configure Datasette's permission system.
+Which resources can the current actor access?
+    Use :ref:`AllowedResourcesView` to view an access map for a selected action.
+
+Which raw rules did Datasette and its plugins contribute?
+    Use :ref:`PermissionRulesView` to inspect the rules before they are resolved into decisions.
+
+Which checks has this Datasette instance performed recently?
+    Use ``/-/permissions`` to view recent permission activity.
+
+These tools are designed to help administrators and plugin authors understand and confirm the effective permissions configuration.
+
+These debug endpoints are exempt from the :ref:`JSON API stability promise <json_api_stability>` - their JSON shapes may change in future releases.
 
 .. _AllowedResourcesView:
 
@@ -1141,7 +1216,7 @@ The ``/-/allowed`` endpoint displays resources that the current actor can access
 
 This endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/allowed.json``) to get the raw JSON response instead.
 
-Pass ``?action=view-table`` (or another action) to select the action. Optional ``parent=`` and ``child=`` query parameters can narrow the results to a specific database/table pair.
+Pass ``?action=view-table`` (or another action) to select the action. Optional ``parent=`` and ``child=`` query parameters can narrow the results to a specific database/table pair. Results are paginated: ``?_size=`` sets the page size (default 50, maximum 200, ``max`` for the maximum) and ``?_page=`` selects a page.
 
 This endpoint is publicly accessible to help users understand their own permissions. The potentially sensitive ``reason`` field is only shown to users with the ``permissions-debug`` permission - it shows the plugins and explanatory reasons that were responsible for each decision.
 
@@ -1154,7 +1229,7 @@ The ``/-/rules`` endpoint displays all permission rules (both allow and deny) fo
 
 This endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/rules.json?action=view-table``) to get the raw JSON response instead.
 
-Pass ``?action=`` as a query parameter to specify which action to check.
+Pass ``?action=`` as a query parameter to specify which action to check. The ``?_size=`` and ``?_page=`` pagination parameters work the same as on ``/-/allowed``.
 
 This endpoint requires the ``permissions-debug`` permission.
 
@@ -1163,11 +1238,20 @@ This endpoint requires the ``permissions-debug`` permission.
 Permission check view
 ---------------------
 
-The ``/-/check`` endpoint evaluates a single action/resource pair and returns information indicating whether the access was allowed along with diagnostic information.
+The ``/-/check`` endpoint evaluates and explains a single actor, action and resource decision. The explanation includes:
+
+* Every matching allow and deny rule, with its source and reason.
+* The winning resource, parent or global scope.
+* Rules ignored because a more specific rule matched, or because a deny won at the same scope.
+* Actor restriction allowlists that included or excluded the resource.
+* Additional actions required by the requested action.
+* An explicit default-deny explanation when no rule matched.
 
 This endpoint provides an interactive HTML form interface. Add ``.json`` to the URL path (e.g. ``/-/check.json?action=view-instance``) to get the raw JSON response instead.
 
-Pass ``?action=`` to specify the action to check, and optional ``?parent=`` and ``?child=`` parameters to specify the resource.
+Pass ``?action=`` to specify the action to check, and optional ``?parent=`` and ``?child=`` parameters to specify the resource. The interactive form also accepts actor JSON, allowing a hypothetical actor to be tested without signing in as that actor. The JSON endpoint accepts the same value using the ``actor`` query string parameter. Use ``actor=null`` to represent an anonymous actor.
+
+This endpoint requires the ``permissions-debug`` permission. The hypothetical actor is used only for the decision being explained; access to the debug tool is checked against the actor who is actually signed in.
 
 .. _authentication_ds_actor:
 
@@ -1369,6 +1453,16 @@ create-table
 
 Actor is allowed to create a database table.
 
+``resource`` - ``datasette.resources.DatabaseResource(database)``
+    ``database`` is the name of the database (string)
+
+.. _actions_create_view:
+
+create-view
+-----------
+
+Actor is allowed to create a database view.
+
 ``resource`` - ``datasette.resources.DatabaseResource(database)``
     ``database`` is the name of the database (string)
 
@@ -1408,6 +1502,18 @@ Actor is allowed to drop a database table.
 
     ``table`` is the name of the table (string)
 
+.. _actions_drop_view:
+
+drop-view
+---------
+
+Actor is allowed to drop a database view.
+
+``resource`` - ``datasette.resources.TableResource(database, table)``
+    ``database`` is the name of the database (string)
+
+    ``table`` is the name of the view (string)
+
 .. _actions_execute_sql:
 
 execute-sql

docs/binary_data.rst

@@ -12,7 +12,12 @@ Datasette includes special handling for these binary values. The Datasette inter
    :width: 311px
    :alt: Screenshot showing download links next to binary data in the table view
 
-Binary data is represented in ``.json`` exports using Base64 encoding.
+.. _binary_json_format:
+
+Binary values in JSON
+---------------------
+
+Binary data is represented in ``.json`` exports using Base64 encoding. Datasette uses this representation for every ``BLOB`` value, including binary values that could also be decoded as UTF-8 text.
 
 https://latest.datasette.io/fixtures/binary_data.json?_shape=array
 
@@ -39,6 +44,48 @@ https://latest.datasette.io/fixtures/binary_data.json?_shape=array
         }
     ]
 
+The same format can be used with the :ref:`JSON write API <json_api_write>`.
+If a column value in a ``row``, ``rows`` or ``update`` object is a JSON object with exactly ``"$base64"`` set to ``true`` and an ``"encoded"`` string, Datasette will decode that Base64 string and store the resulting bytes:
+
+.. code-block:: json
+
+    {
+        "data": {
+            "$base64": true,
+            "encoded": "FRwCx60F/g=="
+        }
+    }
+
+This works for inserts, upserts and updates. It also works when creating a table from example ``row`` or ``rows`` data: Datasette decodes the value before inferring the schema, allowing that column to be created as a ``BLOB`` column.
+
+To store a JSON object with that exact shape literally, wrap it in a ``"$raw"`` object:
+
+.. code-block:: json
+
+    {
+        "data": {
+            "$raw": {
+                "$base64": true,
+                "encoded": "FRwCx60F/g=="
+            }
+        }
+    }
+
+``"$raw"`` unwraps exactly one layer. To store a literal ``"$raw"`` object containing a Base64 object, wrap it again:
+
+.. code-block:: json
+
+    {
+        "data": {
+            "$raw": {
+                "$raw": {
+                    "$base64": true,
+                    "encoded": "FRwCx60F/g=="
+                }
+            }
+        }
+    }
+
 .. _binary_linking:
 
 Linking to binary downloads

docs/changelog.rst

@@ -4,9 +4,121 @@
 Changelog
 =========
 
+.. _v1_0_a37:
+
+1.0a37 (2026-07-14)
+-------------------
+
+Performance improvement for SQL-backed permission checks, plus an improved permission debugging interface.
+
+- SQL used to resolve permission checks now aggregates permission rules before joining them to resources, improving performance on instances with large schemas. (:issue:`2832`)
+- The :ref:`PermissionCheckView` permission debugger now explains why a decision was allowed or denied, including the matching rules. The interactive form can also test a hypothetical actor supplied as JSON, and the :ref:`permissions documentation <authentication_permissions_explained>` now describes resolution rules in more detail. (:issue:`2841`)
+- :ref:`db.execute_write(sql, ..., transaction=True) <database_execute_write>` has a new ``transaction=`` parameter, which can be set to ``False`` for statements such as ``VACUUM`` that cannot run inside a transaction. Write tasks now start their transactions using ``BEGIN IMMEDIATE``, which also ensures that writes are rolled back if the task fails. (:issue:`2831`)
+- Refreshing a database's schema in Datasette's internal catalog is now performed as a single atomic operation. (:issue:`2831`)
+- Fixed schema introspection, table pages, facets and table counts for tables with names containing a ``]`` character. Thanks, `TowyTowy <https://github.com/TowyTowy>`__. (:issue:`2431`, :pr:`2846`)
+- ``/-/plugins.json`` once again returns a top-level JSON array of plugin objects, reverting the object envelope introduced in 1.0a36. This should fix a large number of trivial test failures in existing plugins. (:issue:`2842`, :pr:`2843`)
+
+.. _v1_0_a36:
+
+1.0a36 (2026-07-07)
+-------------------
+
+The signature features of this alpha are new UIs for **inserting multiple rows at once** (from TSV, CSV or JSON) and for **creating a table from rows**, plus a large number of small **JSON API consistency fixes** in preparation for a 1.0 stable release.
+
+- Table pages now offer an "Insert multiple rows" mode in the row insertion dialog. This accepts pasted TSV, CSV or JSON, previews the parsed rows before inserting them, validates unknown columns as data is pasted and displays omitted auto integer primary keys as ``auto`` in the preview. (:pr:`2813`)
+- The bulk insert UI can skip rows with existing primary keys, or update existing rows and insert new rows using the existing ``/<database>/<table>/-/upsert`` API when the actor has both :ref:`insert-row <actions_insert_row>` and :ref:`update-row <actions_update_row>` permissions. (:pr:`2813`)
+- The "Create table" dialog now includes a "Create table from data" mode. Paste TSV, CSV or JSON rows to preview inferred columns and types, choose the table name and primary key, then create the table and insert those rows in one step. (:pr:`2813`)
+- Datasette's JSON APIs now consistently encode every ``BLOB`` value using the documented :ref:`binary value JSON format <binary_json_format>`, even when the bytes could be decoded as UTF-8 text. (:issue:`2806`, :pr:`2822`)
+- The insert and edit row dialogs now provide a dedicated control for ``BLOB`` values. Existing binary values are shown by byte size, image values under 10MB are previewed as thumbnails, and replacements can be attached, dropped or pasted into the control. (:issue:`2806`, :pr:`2822`)
+- The table and row JSON APIs now support ``?_extra=column_details`` for returning SQLite schema details for columns, including declared type, SQLite affinity, primary key, ``NOT NULL``, default and hidden-column metadata.
+- POST bodies that Datasette reads fully into memory - such as JSON submitted to the write API - are now capped by the new :ref:`setting_max_post_body_bytes` setting, defaulting to 2MB. Oversized requests are rejected with an HTTP 413 error as soon as the limit is exceeded, protecting smaller servers from memory exhaustion. File uploads are unaffected - ``request.form()`` streams those to disk and has its own separate limits. (:issue:`2823`)
+- Row pages for tables with compound primary keys now return a ``400`` error instead of a ``500`` error when the URL row identifier does not contain the correct number of primary key values. Thanks, `Zain Dana Harper <https://github.com/HarperZ9>`__. (:issue:`2811`, :pr:`2815`)
+- The :ref:`execute-write-sql <actions_execute_write_sql>` interface now supports ``CREATE VIEW`` and ``DROP VIEW`` statements, gated by the new :ref:`create-view <actions_create_view>` and :ref:`drop-view <actions_drop_view>` permissions. (:issue:`2819`, :pr:`2818`)
+- Saved-query SQL analysis now handles recursive CTEs, fixing a bug where storing a valid read-only recursive query could be disabled by SQLite's internal ``SQLITE_RECURSIVE`` authorizer callback. (:issue:`2809`, :pr:`2812`)
+- ``named_parameters()`` now correctly ignores SQLite comment markers that appear inside string literals, so query forms no longer drop later ``:named`` parameters from SQL such as ``select '--' || :name``. Thanks, `JSap0914 <https://github.com/JSap0914>`__. (:pr:`2783`)
+- Datasette's internal database schema is now managed using `sqlite-utils migrations <https://sqlite-utils.datasette.io/en/stable/python-api.html#migrations>`__, using the new dependency on ``sqlite-utils>=4.0``. (:issue:`2827`)
+- ``datasette.utils.CustomJSONEncoder`` is now documented as a public API for plugins that need to serialize Datasette values to JSON. Thanks, `Chris Amico <https://github.com/eyeseast>`__. (:issue:`1983`, :pr:`1996`)
+
+This release also includes the results of a `detailed consistency review <https://github.com/simonw/datasette/pull/2824>`__ of Datasette's JSON API in preparation for the 1.0 stable release. Several of these changes are backwards-incompatible with previous 1.0 alphas. The new :ref:`API stability documentation <json_api_stability>` describes exactly which parts of the JSON API are covered by the 1.0 stability promise.
+
+JSON API: breaking changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- JSON error responses now use a single canonical format across every endpoint: ``{"ok": false, "error": "...", "errors": [...], "status": 400}``. The ``error`` key joins all error messages together, ``errors`` is the full list of messages and ``status`` always matches the HTTP status code. The legacy ``title`` key is no longer included in JSON errors (it remains available to the HTML error template), and endpoints that previously returned bare ``{"error": ...}`` objects have been updated. See :ref:`json_api_errors`.
+- Every JSON object success response now includes ``"ok": true``, including introspection endpoints such as ``/-/versions`` and ``/-/settings``.
+- ``/-/plugins.json``, ``/-/databases.json`` and ``/-/actions.json`` now return objects - ``{"ok": true, "plugins": [...]}`` and equivalents - instead of top-level JSON arrays, so these responses can gain additional keys in the future without a breaking change. The ``datasette plugins`` CLI command still outputs a plain array.
+- ``/-/databases`` now only lists databases the current actor is allowed to view. It previously listed every attached database, including their filesystem paths, to any actor with ``view-instance``.
+- Requests with an invalid or expired ``Authorization: Bearer`` token now receive a ``401`` status with the standard error body and a ``WWW-Authenticate: Bearer error="invalid_token"`` header, instead of being silently treated as unauthenticated. Bearer tokens that no registered token handler recognizes are still ignored, so authentication plugins with their own token formats keep working. Plugin :ref:`token handlers <plugin_hook_register_token_handler>` can raise the new ``datasette.TokenInvalid`` exception to trigger the same behavior.
+- Permission errors for JSON requests now return the standard JSON error format with a ``403`` status. The default forbidden handling previously rendered an HTML error page even for ``.json`` requests.
+- ``POST`` to a write canned query now returns a ``400`` error when the SQL fails to execute, instead of a ``200`` status with ``"ok": false`` in the body. The error response includes the standard error keys plus a ``"redirect"`` key.
+- The :ref:`row update API <RowUpdateView>` with ``"return": true`` now responds with a ``"rows"`` list, matching insert and upsert, instead of a singular ``"row"`` object.
+- Row delete write failures - such as a constraint violation raised by a trigger - now return ``400`` instead of ``500``, matching the other write endpoints.
+- ``/<database>/-/query.json`` with a missing or blank ``?sql=`` parameter now returns a ``400`` error, as the CSV format already did, instead of a ``200`` with empty rows.
+- Unknown ``?_extra=`` names now return a ``400`` error for JSON and other data formats, instead of being silently ignored. HTML pages continue to ignore unknown names.
+- Table JSON responses now include ``next_url`` alongside ``next`` by default - both are ``null`` on the final page. The now-redundant ``?_extra=next_url`` parameter has been removed.
+- The stored query list JSON no longer includes ``has_more`` - ``"next": null`` is the end-of-results signal across the whole API. This change also uncovered and fixed a bug where the query list ``next_url`` pointed at the HTML page and was a relative path; it is now an absolute URL that preserves the requested format.
+- Stored query JSON objects no longer duplicate the list of parameter names as both ``params`` and ``parameters`` - only ``parameters`` remains. The query create and update APIs no longer accept ``params`` as an input alias either; ``params`` is still the documented key for :ref:`queries defined in configuration <queries_named_parameters>`.
+- Page size parameters are now consistent across the API: the stored query lists accept ``?_size=max`` and return a ``400`` error for values over the maximum instead of silently clamping them, and the ``/-/allowed`` and ``/-/rules`` permission debug endpoints renamed their ``page`` and ``page_size`` parameters to ``_page`` and ``_size``, matching the underscore grammar used by every other Datasette system parameter.
+- ``/-/threads`` now requires the ``permissions-debug`` permission, since it exposes runtime internals such as file paths. It previously only required ``view-instance``.
+- Trusted stored queries - those defined in configuration - can no longer be deleted through the JSON API or web interface, matching the existing restriction on editing them.
+- The ``/<database>/-/schema`` endpoints now check the ``view-database`` permission before checking whether the database exists, so unauthorized actors can no longer probe for the existence of databases.
+- SQL time limit errors in JSON responses are now a plain text message. The error string previously embedded an HTML fragment.
+- The undocumented homepage JSON at ``/.json`` now returns ``databases`` as a list of objects rather than an object keyed by database name, matching every other collection in the API.
+- The legacy ``.jsono`` format extension, long since superseded by ``?_shape=``, has been removed.
+
+JSON API: other improvements
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- The :ref:`write API <json_api_write>` endpoints now parse the request body as JSON regardless of the ``Content-Type`` header, so ``curl -d`` invocations work without remembering to set it. Invalid JSON is a ``400`` error. Cross-site request forgery remains prevented by Datasette's ``Origin`` and ``Sec-Fetch-Site`` checks. This also fixes a ``500`` error from the insert API when the ``Content-Type`` header was missing entirely.
+- New ``Response.error(messages, status=400)`` helper for plugins that need to return a JSON error in Datasette's standard format. See :ref:`internals_response`.
+- New ``count_truncated`` extra for table JSON, included automatically whenever ``count`` is requested. ``true`` means the count reached Datasette's counting limit and the real number of rows may be higher. See :ref:`json_api_extra`.
+- JSON endpoints that are not part of the documented stable API now declare themselves with an ``"unstable"`` key in their responses.
+- New documentation covering the grammar for :ref:`boolean query string arguments <json_api_table_arguments>`, the reason :ref:`upsert <TableUpsertView>` returns ``200`` where insert returns ``201``, and advice for plugin authors on :ref:`naming secret configuration keys <plugins_configuration_secret>` so that ``/-/config`` redacts them automatically.
+
+.. _v1_0_a35:
+
+1.0a35 (2026-06-23)
+-------------------
+
+This release adds UI for **creating tables** and **altering tables**, to complement the insert and update row interfaces added in :ref:`v1_0_a34`.
+
+- New "Create table" interface in the database actions menu, backed by the ``/<database>/-/create`` :ref:`JSON API <TableCreateView>`. It can define columns, primary keys, custom column types, ``NOT NULL`` constraints, literal defaults, expression defaults and single-column foreign keys. (:issue:`2787`)
+- New "Alter table" table action and ``/<database>/<table>/-/alter`` :ref:`JSON API <TableAlterView>` for changing existing tables: add, rename, reorder and drop columns; change column types, defaults, ``NOT NULL`` constraints, primary keys and foreign keys; and rename the table. The alter table dialog also includes a "Drop table" button. (:issue:`2788`)
+- New ``/<database>/-/foreign-key-targets`` and ``/<database>/<table>/-/foreign-key-suggestions`` JSON APIs for discovering valid single-column foreign key targets and suggested relationships.
+- New :ref:`template_context` documentation listing the variables available to custom templates for Datasette's core pages. Variables documented there are treated as a stable API for custom templates until Datasette 2.0. The documentation is generated from dataclass definitions next to the view code, with tests that compare the documented fields against the actual contexts rendered by the database, table, query and row pages. (:issue:`1510`, :issue:`2127`, :issue:`1477`, :pr:`2803`)
+- The "Write to this database" page now includes a Create table starter template, alongside the existing Insert, Update and Delete templates. (:pr:`2794`)
+- New ``static()`` template function and ``datasette.static()`` method for generating cache-busting static asset URLs based on the file contents. Static assets served with a matching ``?_hash=`` parameter now receive far-future immutable cache headers. This works for Datasette's bundled static assets, plugin static assets and directories mounted using ``--static``. See :ref:`customization_static_files`.
+- Database and table pages now use the ``count_truncated`` template context value to display capped row counts as ``>N rows``.
+- Significant visual improvements to the table filter form UI, plus working add/remove filter buttons. (:issue:`2798`)
+- Improved edit row icon on table pages. (:issue:`2796`)
+- Documentation covers how actors are displayed. Thanks, `Sebastian Cao <https://github.com/cycsmail>`__. (:issue:`2002`)
+- Fix for bug where appending ``?_col=pk`` resulted in duplicate primary key columns in the response. Thanks, `Ritesh Kewlani <https://github.com/riteshkew>`__. (:issue:`1975`)
+
+.. _v1_0_a34:
+
+1.0a34 (2026-06-16)
+-------------------
+
+The big feature in this alpha is tools to **insert, edit and delete** rows within the Datasette interface. These features are available on table pages, and edit and delete are also available as action items on the row page.
+
+The edit interface takes :ref:`custom column types <table_configuration_column_types>` into account. Plugins that define their own column types can use JavaScript to customize how those column types are presented in the edit interface.
+
+- ``datasette.allowed_many()`` method for :ref:`resolving multiple permission checks at once <datasette_allowed_many>`. (:pr:`2775`)
+- Permission checks are now cached on a per-request basis, speeding up table pages with multiple plugins that check permissions in order to populate the :ref:`table actions menu <plugin_hook_table_actions>`.
+- Fixed a warning about ``gen.throw(*sys.exc_info())``. (:issue:`2776`)
+- New default custom column type ``textarea`` for multi-line text content. This is rendered as a ``<textarea>`` input in the edit UI.
+- The ``json`` column type now implements client-side validation in the edit UI.
+- The :ref:`makeColumnField() <javascript_plugins_makeColumnField>` JavaScript plugin hook allows plugins to define custom fields in the edit interface for their custom column types.
+- New UI for inserting, editing, and deleting rows within Datasette. (:issue:`2780`)
+- New ``/<database>/<table>/-/autocomplete?q=term`` :ref:`autocomplete JSON API <TableAutocompleteView>` for rapid autocomplete search against the contents of a table. This is used by the edit interface to select related rows for foreign keys. You can try it out on the ``/-/debug/autocomplete`` debug page.
+- New ``/<database>/<table>/-/fragment`` :ref:`HTML fragment endpoint  <TableFragmentView>` for returning the HTML used to display a specific row.
+- ``await request.json()`` utility method for consuming the request body as JSON. (:issue:`2767`)
+- Database, table, query and row action menus can now be modified by plugins to :ref:`display buttons in addition to links <plugin_actions>`. (:issue:`2782`)
+- Datasette :ref:`now uses Playwright <contributing_playwright>` for browser automation tests as part of the test suite. (:issue:`2779`)
+
 .. _v1_0_a33:
 
-1.0a33 (unreleased)
+1.0a33 (2026-06-11)
 -------------------
 
 Stored queries can now be edited and deleted through the web interface, and the JSON API ``?_extra=`` mechanism has been extended to cover row and query pages in addition to tables. This release also fixes two security issues: an identifier-quoting bug involving table and column names that contain ``]``, and an open redirect.

docs/cli-reference.rst

@@ -244,6 +244,9 @@ These can be passed to ``datasette serve`` using ``datasette serve --setting nam
                                    custom query (default=1000)
       max_insert_rows              Maximum rows that can be inserted at a time using
                                    the bulk insert API (default=100)
+      max_post_body_bytes          Maximum size in bytes for a POST body read into
+                                   memory, e.g. JSON API requests - set 0 to disable
+                                   this limit (default=2097152)
       num_sql_threads              Number of threads in the thread pool for
                                    executing SQLite queries (default=3)
       sql_time_limit_ms            Time limit for a SQL query in milliseconds

docs/configuration.rst

@@ -1102,9 +1102,9 @@ These configure :ref:`full-text search <full_text_search>` for a table or view.
 ``column_types``
 ^^^^^^^^^^^^^^^^
 
-You can assign semantic column types to columns, which affect how values are rendered, validated, and transformed. Built-in column types include ``url``, ``email``, and ``json``. Plugins can register additional column types using the :ref:`register_column_types <plugin_register_column_types>` plugin hook.
+You can assign semantic column types to columns, which affect how values are rendered, validated, transformed, and edited. Built-in column types include ``url``, ``email``, ``json``, and ``textarea``. Plugins can register additional column types using the :ref:`register_column_types <plugin_register_column_types>` plugin hook.
 
-Column types can optionally declare which SQLite column types they apply to using ``sqlite_types``. Datasette will reject incompatible assignments. The built-in ``url``, ``email``, and ``json`` column types are all restricted to ``TEXT`` columns.
+Column types can optionally declare which SQLite column types they apply to using ``sqlite_types``. Datasette will reject incompatible assignments. The built-in ``url``, ``email``, ``json``, and ``textarea`` column types are all restricted to ``TEXT`` columns.
 
 The simplest form maps column names to type name strings:
 
@@ -1119,6 +1119,7 @@ The simplest form maps column names to type name strings:
                   website: url
                   contact: email
                   extra_data: json
+                  notes: textarea
       """).strip()
     )
 .. ]]]
@@ -1135,6 +1136,7 @@ The simplest form maps column names to type name strings:
                   website: url
                   contact: email
                   extra_data: json
+                  notes: textarea
 
 .. tab:: datasette.json
 
@@ -1148,7 +1150,8 @@ The simplest form maps column names to type name strings:
                   "column_types": {
                     "website": "url",
                     "contact": "email",
-                    "extra_data": "json"
+                    "extra_data": "json",
+                    "notes": "textarea"
                   }
                 }
               }

docs/contributing.rst

@@ -62,6 +62,76 @@ You can run the tests faster using multiple CPU cores with `pytest-xdist <https:
 
     uv run pytest -m "serial"
 
+.. _contributing_playwright:
+
+Running Playwright tests
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+Datasette includes a small number of browser automation tests using Playwright_.
+These tests are skipped by default, so you can run the main test suite with
+``uv run pytest`` without installing Playwright or any browser binaries.
+
+.. _Playwright: https://playwright.dev/python/
+
+The Playwright tests use a separate dependency group. The easiest way to run
+them is using ``just``. First install the browser engine you want to test
+against. Chromium is used by default:
+
+.. code-block:: bash
+
+    just playwright-install
+
+Then run the Playwright test module:
+
+.. code-block:: bash
+
+    just playwright
+
+You can also run the same tests against Firefox or WebKit by installing that
+browser engine and passing it to ``just playwright``:
+
+.. code-block:: bash
+
+    just playwright-install firefox
+    just playwright firefox
+
+    just playwright-install webkit
+    just playwright webkit
+
+To install every supported browser engine and run the tests against all of
+them, use:
+
+.. code-block:: bash
+
+    just playwright-install-all
+    just playwright-all
+
+You can pass extra ``pytest`` options after the browser name:
+
+.. code-block:: bash
+
+    just playwright chromium -k permissions
+    just playwright-all -x
+
+You can add the ``--headed`` option to have Playwright open a browser window that you can see while it runs the tests. This only works if you specify a browser, for example:
+
+.. code-block:: bash
+
+    just playwright firefox --headed
+
+Combine this with ``-k`` to watch a specific test:
+
+.. code-block:: bash
+
+    just playwright chromium --headed -k test_insert_row
+
+If you are not using ``just``, the equivalent ``uv run`` commands are:
+
+.. code-block:: bash
+
+    uv run --group playwright playwright install chromium
+    uv run --group playwright pytest tests/test_playwright.py --playwright --browser chromium
+
 .. _contributing_using_fixtures:
 
 Using fixtures
@@ -87,6 +157,9 @@ If you want to change Datasette's Python code you can use the ``--reload`` optio
 
     uv run datasette --reload fixtures.db
 
+This also enables development mode for static asset cache busting, described in
+:ref:`customization_static_files`.
+
 You can also use the ``fixtures.py`` script to recreate the testing version of ``metadata.json`` used by the unit tests. To do that::
 
     uv run python tests/fixtures.py fixtures.db fixtures-metadata.json
@@ -239,6 +312,19 @@ To update these pages, run the following command::
 
     uv run cog -r docs/*.rst
 
+.. _contributing_template_contexts:
+
+Documented template contexts
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Datasette's documented template contexts are part of the public API for custom templates. They are defined as dataclasses next to the view code that renders them, for example ``DatabaseContext`` and ``QueryContext`` in ``datasette/views/database.py``.
+
+Every documented context class inherits from ``datasette.views.Context``. Fields that are added directly by view code should be declared as dataclass fields with ``help`` metadata, which is used to generate :ref:`template_context`. Fields resolved through the page extras system should use ``from_extra()`` so their documentation comes from the matching ``Extra`` class.
+
+Use ``documented_template`` on each context class to record the canonical template named in the generated documentation. This should be a string such as ``"database.html"``. Runtime template selection still happens in the view code, since most pages consider more specific template names before falling back to the canonical one.
+
+When a context field contains repeated structured data, prefer a small nested dataclass over an anonymous dictionary. For example, a field containing table summaries should be annotated as ``list[DatabaseTable]`` where ``DatabaseTable`` is a dataclass describing the keys and value types. This keeps the Python contract and generated documentation clear. JSON responses and ``?_context=1`` debug output will convert nested dataclasses back to JSON objects at the response boundary.
+
 .. _contributing_continuous_deployment:
 
 Continuously deployed demo instances

docs/custom_templates.rst

@@ -94,11 +94,55 @@ The ``core`` class is particularly useful - you can apply this directly to a ``<
 
 .. _customization_static_files:
 
-Serving static files
-~~~~~~~~~~~~~~~~~~~~
+Linking to static assets
+~~~~~~~~~~~~~~~~~~~~~~~~
 
-Datasette can serve static files for you, using the ``--static`` option.
-Consider the following directory structure::
+Use the ``static()`` template function to create cache-busting URLs to static
+assets from your custom templates. It returns a URL with a ``?_hash=`` parameter
+based on the file contents and takes the ``base_url`` setting into account.
+
+When the hash in the URL matches the current file contents, Datasette will serve
+the static asset with a far-future immutable ``Cache-Control`` header.
+
+When Datasette is run using ``--reload``, the file contents are hashed every time
+the template is rendered, so edits to static files will update their URLs
+without restarting Datasette. Without ``--reload``, the hash is cached for the
+lifetime of the Datasette process.
+
+For Datasette's bundled static assets, pass the path to the asset:
+
+.. code-block:: html+jinja
+
+    <link rel="stylesheet" href="{{ static('app.css') }}">
+
+For plugin static assets, pass the plugin name using ``plugin=`` and a path
+relative to the plugin's ``static/`` directory:
+
+.. code-block:: html+jinja
+
+    <script src="{{ static('plugin.js', plugin='datasette_plugin_name') }}" defer></script>
+
+For files served from a directory mounted using ``--static assets:static-files/``,
+pass the mount point name using ``mount=`` and a path relative to that mounted
+directory:
+
+.. code-block:: html+jinja
+
+    <link rel="stylesheet" href="{{ static('styles.css', mount='assets') }}">
+    <script src="{{ static('app.js', mount='assets') }}" defer></script>
+
+You can also use ``urls.path()`` if you want to link to a mounted file without
+adding a content hash:
+
+.. code-block:: html+jinja
+
+    <link rel="stylesheet" href="{{ urls.path('/assets/styles.css') }}">
+
+Serving files from a directory
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Datasette can serve static files from a directory for you, using the ``--static``
+option. Consider the following directory structure::
 
     metadata.json
     static-files/styles.css
@@ -177,6 +221,11 @@ this::
 Datasette will now first look for templates in that directory, and fall back on
 the defaults if no matches are found.
 
+The variables made available to each template are documented on the
+:ref:`template_context` page. Variables documented there are a stable API:
+custom templates that use them will keep working in future Datasette
+releases, up until the next major version.
+
 It is also possible to over-ride templates on a per-database, per-row or per-
 table basis.
 
@@ -274,13 +323,28 @@ Here is an example of a custom ``_table.html`` template:
 .. code-block:: jinja
 
     {% for row in display_rows %}
-        <div>
+        <div data-row="{{ row.row_path }}">
             <h2>{{ row["title"] }}</h2>
             <p>{{ row["description"] }}<lp>
             <p>Category: {{ row.display("category_id") }}</p>
         </div>
     {% endfor %}
 
+If your custom table template should support Datasette's row editing UI, include
+``data-row="{{ row.row_path }}"`` on the outer element that represents each row.
+This does not need to be a ``<tr>``: it can be a ``<div>``, ``<li>`` or any other
+element that wraps the HTML for that row. Datasette uses this attribute to find
+the element to remove after a delete, or replace after an edit. Any edit or
+delete controls should be rendered inside that same element.
+
+The ``_action_menu.html`` template renders the action menus used by database,
+table, query and row pages. Plugin-provided actions can be link dictionaries
+with ``href`` and ``label`` keys, or button dictionaries using ``{"type":
+"button", "label": "...", "attrs": {...}}`` for JavaScript-backed interactions.
+Both shapes can include an optional ``description`` key. Custom
+``_action_menu.html`` templates should preserve support for both link and button
+action items.
+
 .. _custom_pages:
 
 Custom pages

docs/index.rst

@@ -58,6 +58,7 @@ Contents
    settings
    introspection
    custom_templates
+   template_context
    plugins
    writing_plugins
    javascript_plugins

docs/installation.rst

@@ -17,13 +17,6 @@ If you want to start making contributions to the Datasette project by installing
 Basic installation
 ==================
 
-.. _installation_datasette_desktop:
-
-Datasette Desktop for Mac
--------------------------
-
-`Datasette Desktop <https://datasette.io/desktop>`__ is a packaged Mac application which bundles Datasette together with Python and allows you to install and run Datasette directly on your laptop. This is the best option for local installation if you are not comfortable using the command line.
-
 .. _installation_homebrew:
 
 Using Homebrew

docs/internals.rst

@@ -52,6 +52,9 @@ The request object is passed to various plugin hooks. It represents an incoming
 ``.actor`` - dictionary (str -> Any) or None
     The currently authenticated actor (see :ref:`actors <authentication_actor>`), or ``None`` if the request is unauthenticated.
 
+``.max_post_body_bytes`` - integer
+    The maximum number of bytes ``await request.post_body()`` will read into memory, or ``0`` for no limit. Set from the :ref:`setting_max_post_body_bytes` setting (default 2MB) for requests created by Datasette. Can be passed to the ``Request`` constructor as a keyword argument.
+
 The object also has the following awaitable methods:
 
 ``await request.form(files=False, ...)`` - FormData
@@ -106,9 +109,14 @@ The object also has the following awaitable methods:
 ``await request.post_vars()`` - dictionary
     Returns a dictionary of form variables that were submitted in the request body via ``POST`` using ``application/x-www-form-urlencoded`` encoding. For multipart forms or file uploads, use ``request.form()`` instead.
 
-``await request.post_body()`` - bytes
+``await request.json()`` - Any
+    Returns the parsed JSON body of a request submitted by ``POST``.
+
+``await request.post_body(max_bytes=None)`` - bytes
     Returns the un-parsed body of a request submitted by ``POST`` - useful for things like incoming JSON data.
 
+    The body is read fully into memory, capped at ``request.max_post_body_bytes`` - which Datasette sets from the :ref:`setting_max_post_body_bytes` setting (default 2MB). Bodies that exceed the limit raise a ``datasette.PayloadTooLarge`` exception, which Datasette turns into an HTTP 413 error response. Pass ``max_bytes=`` to override the limit for a specific call, or ``max_bytes=0`` to disable it. ``request.post_vars()`` and ``request.json()`` read the body through this method, so the same limit applies to them.
+
 And a class method that can be used to create fake request objects for use in tests:
 
 ``fake(path_with_query_string, method="GET", scheme="http", url_vars=None)``
@@ -276,7 +284,7 @@ For example:
         content_type="application/xml; charset=utf-8",
     )
 
-The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)`` or ``Response.redirect(...)`` helper methods:
+The quickest way to create responses is using the ``Response.text(...)``, ``Response.html(...)``, ``Response.json(...)``, ``Response.error(...)`` or ``Response.redirect(...)`` helper methods:
 
 .. code-block:: python
 
@@ -287,6 +295,8 @@ The quickest way to create responses is using the ``Response.text(...)``, ``Resp
     text_response = Response.text(
         "This will become utf-8 encoded text"
     )
+    # A JSON error in Datasette's standard error format:
+    error_response = Response.error("Cannot do that", 400)
     # Redirects are served as 302, unless you pass status=301:
     redirect_response = Response.redirect(
         "https://latest.datasette.io/"
@@ -296,6 +306,8 @@ Each of these responses will use the correct corresponding content-type - ``text
 
 Each of the helper methods take optional ``status=`` and ``headers=`` arguments, documented above.
 
+``Response.error(messages, status=400)`` returns a JSON error in the :ref:`standard Datasette error format <json_api_errors>`. ``messages`` can be a single string or a list of strings. Use this for JSON-only endpoints; if your error should content-negotiate between JSON and HTML, raise ``Forbidden``, ``NotFound``, ``BadRequest`` or ``DatasetteError`` instead and Datasette's error handling will build the appropriate response.
+
 .. _internals_response_asgi_send:
 
 Returning a response with .asgi_send(send)
@@ -448,6 +460,52 @@ await .render_template(template, context=None, request=None)
 
 Renders a `Jinja template <https://jinja.palletsprojects.com/en/2.11.x/>`__ using Datasette's preconfigured instance of Jinja and returns the resulting string. The template will have access to Datasette's default template functions and any functions that have been made available by other plugins.
 
+.. _datasette_static:
+
+.static(path, plugin=None, mount=None)
+--------------------------------------
+
+``path`` - string
+    The path to the static asset, relative to the selected static directory.
+
+``plugin`` - string, optional
+    The plugin name, for linking to an asset in that plugin's ``static/``
+    directory.
+
+``mount`` - string, optional
+    The ``--static`` mount name, for linking to an asset in a directory mounted
+    using ``datasette --static mount_name:directory``.
+
+Returns a URL for a static asset with a ``?_hash=`` parameter based on the file
+contents. That URL takes the ``base_url`` setting into account.
+
+When the ``?_hash=`` parameter matches the current file contents, Datasette will
+serve the asset with ``Cache-Control: max-age=31536000, immutable, public``.
+
+Call this with just ``path`` for one of Datasette's bundled static assets:
+
+.. code-block:: python
+
+    datasette.static("app.css")
+
+Use ``plugin=`` for plugin static assets:
+
+.. code-block:: python
+
+    datasette.static(
+        "plugin.js", plugin="datasette_plugin_name"
+    )
+
+Use ``mount=`` for static directories mounted using the ``--static`` option:
+
+.. code-block:: python
+
+    datasette.static("styles.css", mount="assets")
+
+``plugin`` and ``mount`` are mutually exclusive. The same feature is available
+to Jinja templates as the ``static()`` template function, described in
+:ref:`customization_static_files`.
+
 .. _datasette_actors_from_ids:
 
 await .actors_from_ids(actor_ids)
@@ -512,6 +570,43 @@ Example usage:
 
 The method returns ``True`` if the permission is granted, ``False`` if denied.
 
+Results are cached for the duration of the current request, so checking the same ``(actor, action, resource)`` combination twice within one request only does the underlying permission resolution work once.
+
+.. _datasette_allowed_many:
+
+await .allowed_many(\*, actions, resource, actor=None)
+------------------------------------------------------
+
+``actions`` - list of strings
+    The names of the actions to permission check.
+
+``resource`` - Resource object
+    A Resource object representing the database, table, or other resource that each action is checked against. Omit for global actions.
+
+``actor`` - dictionary, optional
+    The authenticated actor. This is usually ``request.actor``. Defaults to ``None`` for unauthenticated requests.
+
+Checks several actions against the same resource for the same actor, returning a dictionary mapping each action name to ``True`` or ``False``. The whole batch - including any actions pulled in through ``also_requires`` dependencies - is resolved with a single SQL query against the internal database, so this is much faster than calling :ref:`datasette.allowed() <datasette_allowed>` once per action.
+
+Example usage:
+
+.. code-block:: python
+
+    from datasette.resources import TableResource
+
+    results = await datasette.allowed_many(
+        actions=["insert-row", "delete-row", "drop-table"],
+        resource=TableResource(
+            database="fixtures", table="facetable"
+        ),
+        actor=request.actor,
+    )
+    # {"insert-row": True, "delete-row": True, "drop-table": False}
+
+Each result is stored in the per-request permission check cache, so subsequent ``datasette.allowed()`` calls for the same checks within the same request are served from that cache. Datasette uses this before running the ``table_actions`` and ``database_actions`` plugin hooks: it resolves every registered table-level action against the current table and every database-level action against its database first, which means ``allowed()`` calls made by those plugin hooks are usually served from the cache instead of triggering additional queries.
+
+Actions for which no plugin provides any permission rules are resolved to ``False`` directly, without being included in the SQL query at all.
+
 .. _datasette_allowed_resources:
 
 await .allowed_resources(action, actor=None, \*, parent=None, include_is_private=False, include_reasons=False, limit=100, next=None)
@@ -1928,8 +2023,8 @@ Example usage:
 
 .. _database_execute_write:
 
-await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10)
---------------------------------------------------------------------------------------------------------
+await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True)
+--------------------------------------------------------------------------------------------------------------------------
 
 SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received.
 
@@ -1964,7 +2059,9 @@ If you need to retrieve every row returned by a statement, pass ``return_all=Tru
 
 If you pass ``block=False`` this behavior changes to "fire and forget" - queries will be added to the write queue and executed in a separate thread while your code can continue to do other things. The method will return a UUID representing the queued task.
 
-Each call to ``execute_write()`` will be executed inside a transaction.
+Each call to ``execute_write()`` will be executed inside a transaction. Pass
+``transaction=False`` for statements such as ``VACUUM`` that cannot run inside
+a transaction.
 
 .. _database_execute_write_script:
 
@@ -2268,6 +2365,14 @@ The internal database schema is as follows:
 
 .. code-block:: sql
 
+    CREATE TABLE "_sqlite_migrations" (
+       "id" INTEGER PRIMARY KEY,
+       "migration_set" TEXT,
+       "name" TEXT,
+       "applied_at" TEXT
+    );
+    CREATE UNIQUE INDEX "idx__sqlite_migrations_migration_set_name"
+        ON "_sqlite_migrations" ("migration_set", "name");
     CREATE TABLE catalog_databases (
         database_name TEXT PRIMARY KEY,
         path TEXT,
@@ -2493,6 +2598,13 @@ Async version of :ref:`call_with_supported_arguments <internals_utils_call_with_
 
 .. _internals_tracer:
 
+JSON encoding
+-------------
+
+.. _internals_utils_CustomJSONEncoder:
+
+.. autoclass:: datasette.utils.CustomJSONEncoder
+
 datasette.tracer
 ================
 

docs/introspection.rst

@@ -7,6 +7,10 @@ Datasette includes some pages and JSON API endpoints for introspecting the curre
 
 Each of these pages can be viewed in your browser. Add ``.json`` to the URL to get back the contents as JSON.
 
+JSON responses that return an object include an ``"ok": true`` key, consistent with the rest of the :ref:`JSON API <json_api>`.
+
+The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise <json_api_stability>`, with the exception of the debug endpoints ``/-/threads`` and ``/-/actions``, whose shapes may change in future releases.
+
 .. _JsonDataView_metadata:
 
 /-/metadata
@@ -37,6 +41,7 @@ Shows the version of Datasette, Python and SQLite. `Versions example <https://la
 .. code-block:: json
 
     {
+        "ok": true,
         "datasette": {
             "version": "0.60"
         },
@@ -97,6 +102,7 @@ Shows the :ref:`settings` for this instance of Datasette. `Settings example <htt
 .. code-block:: json
 
     {
+        "ok": true,
         "default_facet_size": 30,
         "default_page_size": 100,
         "facet_suggest_time_limit_ms": 50,
@@ -115,6 +121,7 @@ Shows the :ref:`configuration <configuration>` for this instance of Datasette. T
 .. code-block:: json
 
     {
+        "ok": true,
         "settings": {
             "template_debug": true,
             "trace_debug": true,
@@ -129,20 +136,47 @@ Any keys that include the one of the following substrings in their names will be
 /-/databases
 ------------
 
-Shows currently attached databases. `Databases example <https://latest.datasette.io/-/databases>`_:
+Shows currently attached databases that the current actor is allowed to view, based on the ``view-database`` permission. `Databases example <https://latest.datasette.io/-/databases>`_:
 
 .. code-block:: json
 
-    [
-        {
-            "hash": null,
-            "is_memory": false,
-            "is_mutable": true,
-            "name": "fixtures",
-            "path": "fixtures.db",
-            "size": 225280
-        }
-    ]
+    {
+        "ok": true,
+        "databases": [
+            {
+                "hash": null,
+                "is_memory": false,
+                "is_mutable": true,
+                "name": "fixtures",
+                "path": "fixtures.db",
+                "size": 225280
+            }
+        ]
+    }
+
+.. _JsonDataView_actions:
+
+/-/actions
+----------
+
+Shows all actions registered with the permission system, including those added by plugins. Requires the ``permissions-debug`` permission.
+
+.. code-block:: json
+
+    {
+        "ok": true,
+        "actions": [
+            {
+                "name": "view-instance",
+                "abbr": "vi",
+                "description": "View Datasette instance",
+                "takes_parent": false,
+                "takes_child": false,
+                "resource_class": null,
+                "also_requires": null
+            }
+        ]
+    }
 
 .. _JumpView:
 
@@ -160,6 +194,7 @@ The endpoint supports a ``?q=`` query parameter for filtering items by name.
 .. code-block:: json
 
     {
+        "ok": true,
         "matches": [
             {
                 "name": "fixtures",
@@ -188,6 +223,7 @@ Search example with ``?q=facet`` returns only items matching ``.*facet.*``:
 .. code-block:: json
 
     {
+        "ok": true,
         "matches": [
             {
                 "name": "fixtures: facetable",
@@ -201,16 +237,26 @@ Search example with ``?q=facet`` returns only items matching ``.*facet.*``:
 
 When multiple search terms are provided (e.g., ``?q=user+profile``), items must match the pattern ``.*user.*profile.*``. Results are ordered by relevance, then by item type and shortest display name.
 
+.. _AutocompleteDebugView:
+
+/-/debug/autocomplete
+---------------------
+
+The debug tool at ``/-/debug/autocomplete`` can be used to try out the autocomplete component against a specific table. Pass ``?database=db&table=table`` to display an autocomplete field backed by that table's ``/-/autocomplete`` endpoint.
+
+Without those query string arguments, the page lists up to five tables with detected label columns, scanning at most 100 tables.
+
 .. _JsonDataView_threads:
 
 /-/threads
 ----------
 
-Shows details of threads and ``asyncio`` tasks. `Threads example <https://latest.datasette.io/-/threads>`_:
+Shows details of threads and ``asyncio`` tasks. This endpoint requires the ``permissions-debug`` permission, since it exposes runtime internals. `Threads example <https://latest.datasette.io/-/threads>`_:
 
 .. code-block:: json
 
     {
+        "ok": true,
         "num_threads": 2,
         "threads": [
             {
@@ -242,6 +288,7 @@ Shows the currently authenticated actor. Useful for debugging Datasette authenti
 .. code-block:: json
 
     {
+        "ok": true,
         "actor": {
             "id": 1,
             "username": "some-user"

docs/javascript_plugins.rst

@@ -46,6 +46,9 @@ The ``datasetteManager`` object
 ``registerPlugin(name, implementation)``
     Call this to register a plugin, passing its name and implementation
 
+``makeColumnField(context)``
+    Calls the ``makeColumnField()`` hook on registered plugins, returning the first custom insert/edit field control that matches the provided field context. This is used internally by Datasette's row insert and edit dialogs.
+
 ``selectors`` - object
     An object providing named aliases to useful CSS selectors, :ref:`listed below <javascript_datasette_manager_selectors>`
 
@@ -60,22 +63,26 @@ The ``implementation`` object passed to this method should include a ``version``
 
 .. _javascript_plugins_makeJumpSections:
 
-makeJumpSections()
-~~~~~~~~~~~~~~~~~~
+makeJumpSections(context)
+~~~~~~~~~~~~~~~~~~~~~~~~~
 
 This method should return a JavaScript array of objects defining additional sections to be added to the blank state of the ``/`` jump menu, before the user starts typing a search.
 
-Each object should have the following:
+It should return an array of objects, each with the following:
 
 ``id`` - string
     A unique string ID for the section, for example ``agent-chat``
 ``render(node, context)`` - function
     A function that will be called with a DOM node to render the section into
 
-The ``context`` object has the following keys:
+Datasette passes a ``context`` object to both ``makeJumpSections(context)`` and ``render(node, context)``. It has the following keys:
 
 ``navigationSearch``
     The ``<navigation-search>`` custom element instance.
+``container`` - only for ``render()``
+    The ``.results-container`` element used by the jump menu.
+``input`` - only for ``render()``
+    The ``.search-input`` element used by the jump menu.
 
 This example shows how a plugin might add a button for starting a new chat:
 
@@ -84,11 +91,11 @@ This example shows how a plugin might add a button for starting a new chat:
     document.addEventListener('datasette_init', function(ev) {
       ev.detail.registerPlugin('agent-plugin', {
         version: 0.1,
-        makeJumpSections: () => {
+        makeJumpSections: (context) => {
           return [
             {
               id: 'agent-chat',
-              render: node => {
+              render: (node, context) => {
                 node.innerHTML = '<button type="button">Start a new chat</button>';
                 node.querySelector('button').addEventListener('click', () => {
                   location.href = '/-/agent/new';
@@ -188,6 +195,285 @@ This example plugin adds two menu items - one to copy the column name to the cli
       });
     });
 
+.. _javascript_plugins_makeColumnField:
+
+makeColumnField(context)
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+This method, if present, can provide a custom form field for a column in Datasette's row insert and edit dialogs.
+
+It is designed for plugins that :ref:`register custom column types <plugin_register_column_types>` using the Python ``register_column_types()`` plugin hook. For example, a plugin that defines a ``file`` column type can use ``makeColumnField()`` to replace a plain text input with a file picker, and a plugin that defines a rich text column type can use it to enhance the field with an editor.
+
+Datasette calls ``makeColumnField(context)`` on each registered JavaScript plugin when it renders an editable insert/edit field. Plugins should inspect the ``context`` object and only return a control object if they can handle that field. Otherwise, use a bare ``return;``.
+
+The first plugin to return a truthy control object is used for that field. Plugins are called in registration order. If a plugin raises an exception, Datasette logs the error to the browser console and continues to the next plugin.
+
+The row dialog tracks the value that will be sent to the insert/update API. The ``context`` object describes the column and form environment; custom controls should read and write field values using the ``field`` helper object passed to ``render(field)``.
+
+Context object
+^^^^^^^^^^^^^^
+
+``makeColumnField(context)`` is called with a context object describing the field. The current context object has these keys:
+
+``mode`` - string
+    ``"insert"`` or ``"edit"``.
+
+``database`` - string or null
+    The database name.
+
+``table`` - string or null
+    The table name.
+
+``tableUrl`` - string or null
+    The path to the table page, including any configured :ref:`base URL prefix <setting_base_url>`.
+
+``column`` - string
+    The column name.
+
+``columnType`` - object or null
+    The configured Datasette column type for this column, if one exists. This is ``null`` if no column type has been configured.
+
+    If present, this object has exactly these keys:
+
+    ``type`` - string
+        The :ref:`registered column type name <plugin_register_column_types>`, matching the ``name`` attribute of the Python ``ColumnType`` subclass.
+
+    ``config`` - object
+        Configuration for this specific column type assignment. This is ``{}`` if no configuration has been set.
+
+``sqliteType`` - string or null
+    The SQLite affinity for this column, if known. This is one of ``"TEXT"``, ``"INTEGER"``, ``"REAL"``, ``"BLOB"``, ``"NUMERIC"`` or ``null`` if Datasette could not determine the affinity.
+
+``notNull`` - boolean
+    True if the column is defined as ``NOT NULL``.
+
+``isPk`` - boolean
+    True if this column is part of the table's primary key.
+
+``defaultExpression`` - string or null
+    The SQLite default expression for the column, if available. This is ``null`` if the column has no SQLite default. For example, a column defined with ``DEFAULT (datetime('now'))`` will have ``"datetime('now')"`` here. This is the expression from the table schema, not the actual value SQLite will insert.
+
+``form`` - ``HTMLFormElement`` or null
+    The row insert/edit form element.
+
+``dialog`` - ``HTMLDialogElement`` or null
+    The modal dialog element.
+
+Returned control object
+^^^^^^^^^^^^^^^^^^^^^^^
+
+A plugin that wants to handle a field should return an object. Datasette currently recognizes these properties:
+
+``useTextarea`` - boolean, optional
+    If true, Datasette creates a ``<textarea>`` as the underlying ``field.input`` before calling ``render()``. If omitted, Datasette chooses either an ``<input>`` or ``<textarea>`` based on the column type and current value.
+
+``render(field)`` - function
+    Called once to render the custom field UI. ``field`` is a helper object described below.
+
+    The recommended pattern is to return a DOM node from ``render()``. Datasette appends that node to ``field.root``, a ``<div>`` inside the control area for that field in the row insert/edit dialog. A plugin can alternatively manipulate ``field.root`` directly and return nothing.
+
+``focus(field)`` - function, optional
+    Called when Datasette wants to focus this field, for example when focusing the first editable field in the dialog. Use this to focus the most useful interactive element inside the custom UI.
+
+``destroy(field)`` - function, optional
+    Called when Datasette tears down the insert/edit form. Use this to remove event listeners, close nested pickers, revoke object URLs, clear timers, or release other resources.
+
+The field helper object
+^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``field`` object passed to ``render(field)``, ``focus(field)`` and ``destroy(field)`` provides stable IDs, DOM elements and value helpers for integrating with the row insert/edit dialog:
+
+``context`` - object
+    The original context object passed to ``makeColumnField()``.
+
+``id`` - string
+    The ID Datasette assigned to ``field.input``, the backing ``<input>`` or ``<textarea>`` element.
+
+``labelId`` - string
+    The ID of the visible field label.
+
+``descriptionId`` - string
+    The ID of the field metadata/help text. This metadata can include details such as ``Primary key``, ``Required``, ``Current value: NULL`` or ``Custom type: file``.
+
+``root`` - ``HTMLElement``
+    The empty ``<div>`` container created by Datasette for this custom field. It is inside the control area for the field in the row insert/edit dialog, next to the field label and above the field metadata. Datasette appends the DOM node returned by ``render(field)`` to this element. Plugins can alternatively manipulate this element directly and return nothing from ``render(field)``.
+
+``input`` - ``HTMLInputElement`` or ``HTMLTextAreaElement``
+    The core-owned backing form control. Plugins can keep this visible, wrap it or hide it, but should use the value helper methods below rather than mutating ``input.value`` directly.
+
+``control``
+    An alias for ``input``.
+
+``meta`` - ``HTMLElement`` or null
+    The field metadata/help text element.
+
+``form`` - ``HTMLFormElement`` or null
+    The containing row insert/edit form.
+
+``dialog`` - ``HTMLDialogElement`` or null
+    The containing modal dialog.
+
+``getValue()`` - function
+    Returns the current value for this field.
+
+    Datasette uses string values by default. Insert fields for ``"INTEGER"`` and ``"REAL"`` SQLite columns return numbers, or ``null`` if left blank. Plugins can use strings, numbers, booleans or ``null``. If a plugin is editing structured data stored in a SQLite ``TEXT`` column, such as JSON, it should serialize that data to a string before calling ``setValue()``.
+
+``setValue(value)`` - function
+    Sets the current value for this field. ``value`` should be a string, number, boolean or ``null``.
+
+    Calling ``setValue()`` also stops using the SQLite default for the field, if it was previously selected.
+
+``getInitialValue()`` - function
+    Returns the submitted-value representation the field had when the form was rendered. For edit forms this is the raw row value from the database. For insert forms this is the blank starting value.
+
+``hasChanged()`` - function
+    Returns true if the field has changed since Datasette last considered it unmodified. By default that means the field state when the insert/edit form was rendered.
+
+``clearValue()`` - function
+    Sets the value to ``null``.
+
+``markClean()`` - function
+    Tells Datasette to treat the field's current state as unmodified. After calling this method, ``hasChanged()`` returns false until the field value changes again or its SQLite-default state changes.
+
+    This is useful when a plugin rewrites the starting value into an equivalent representation while initializing its editor. For example, a rich text editor might normalize empty HTML or reserialize its initial document before the user has made any edits.
+
+``isUsingSqliteDefault()`` - function
+    Returns true if the insert dialog is currently set to omit this column and use the SQLite default.
+
+``setValidity(message)`` - function
+    Sets a custom validation message for this field, marks the backing input with ``aria-invalid="true"`` and shows the message in the field metadata area. Pass an empty string to clear the error.
+
+``clearValidity()`` - function
+    Clears any custom validation message previously set by ``setValidity()``.
+
+Submitted value contract
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+The ``field.setValue()`` method accepts the following value types:
+
+* string
+* number
+* boolean
+* ``null``
+
+These values are used as column values in requests to the :ref:`insert rows <TableInsertView>` and :ref:`update row <RowUpdateView>` JSON APIs.
+
+Plugins should not pass objects or arrays to ``field.setValue()``. If a column stores structured data in SQLite, such as JSON in a ``TEXT`` column, the plugin should serialize that data first and submit the serialized string. Client-side parsing can still be useful for validation or editor state, but the submitted value should match the SQLite value Datasette should write.
+
+Value helpers
+^^^^^^^^^^^^^
+
+Custom fields should use ``field.getValue()`` and ``field.setValue(value)`` for value handling:
+
+.. code-block:: javascript
+
+    const currentValue = field.getValue();
+    field.setValue("new value");
+    field.setValue(null);
+
+Plugins can keep the core input visible, wrap it in a custom element, or hide it and provide a richer interface. If the input is hidden, the custom UI must still expose an accessible name, state and keyboard interaction.
+
+``field.setValue()`` updates both ``field.input`` and the value used in the insert/update request.
+
+For example, a file picker that stores a selected file ID can hide the backing input and call ``field.setValue()`` when the selection changes:
+
+.. code-block:: javascript
+
+    field.input.type = "hidden";
+    field.setValue(fileId);
+
+For insert forms with a SQLite default, ``field.isUsingSqliteDefault()`` indicates whether Datasette will omit that column from the insert payload. Calling ``field.setValue(value)`` automatically stops using the SQLite default.
+
+Lazy loading large controls
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The JavaScript file that registers ``makeColumnField()`` should be small. If the actual control is large, load it from inside ``render()`` using dynamic ``import()``. That way the heavier code is only downloaded after a user opens an insert/edit dialog containing a matching column type.
+
+.. code-block:: javascript
+
+    const editorUrl = new URL("./editor.js", import.meta.url).href;
+
+    document.addEventListener("datasette_init", function (event) {
+      event.detail.registerPlugin("my-editor", {
+        version: "0.1",
+
+        makeColumnField(context) {
+          if (!context.columnType || context.columnType.type !== "my-editor") {
+            return;
+          }
+          return {
+            useTextarea: true,
+            render(field) {
+              import(editorUrl).then(function () {
+                // Enhance field.input here.
+              });
+              return field.input;
+            }
+          };
+        }
+      });
+    });
+
+Example: textarea-backed custom element
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+This example handles a ``markdown-editor`` column type by asking Datasette for a textarea and wrapping that textarea in a custom ``<my-markdown-editor>`` Web Component element:
+
+.. code-block:: javascript
+
+    document.addEventListener("datasette_init", function (event) {
+      event.detail.registerPlugin("markdown-editor", {
+        version: "0.1",
+
+        makeColumnField(context) {
+          if (!context.columnType || context.columnType.type !== "markdown-editor") {
+            return;
+          }
+
+          return {
+            useTextarea: true,
+
+            render(field) {
+              const editor = document.createElement("my-markdown-editor");
+              editor.appendChild(field.input);
+
+              if (field.labelId) {
+                field.input.setAttribute("aria-labelledby", field.labelId);
+              }
+              if (field.descriptionId) {
+                field.input.setAttribute("aria-describedby", field.descriptionId);
+              }
+
+              return editor;
+            },
+
+            focus(field) {
+              const editor = field.root.querySelector("my-markdown-editor");
+              if (editor && editor.focus) {
+                editor.focus();
+              } else {
+                field.input.focus();
+              }
+            }
+          };
+        }
+      });
+    });
+
+Accessibility
+^^^^^^^^^^^^^
+
+Custom fields are responsible for preserving the accessibility of the form:
+
+- The visible field label should name the control. Use ``field.labelId`` with ``aria-labelledby`` when wrapping or replacing the visible input.
+- Field metadata should remain available to assistive technology. Use ``field.descriptionId`` with ``aria-describedby``.
+- Keyboard users must be able to operate every part of the custom field.
+- If the field opens an inline picker or other nested UI, ``Escape`` should close that nested UI first and return focus to a sensible element.
+- If a control performs asynchronous loading, expose loading and error states in the UI. Use appropriate ARIA live regions where the state change is important to understand the field.
+- If a plugin hides ``field.input``, the replacement UI must still make the current value and available actions clear.
+
+Plugins should not submit the row themselves from inside ``makeColumnField()`` controls. Datasette owns the insert/edit dialog lifecycle, form submission, API call, error handling and row refresh.
+
 .. _javascript_datasette_manager_selectors:
 
 Selectors

docs/json_api_doc.py

@@ -93,9 +93,26 @@ async def _fetch_live_examples(scoped_classes):
         datasette = Datasette(
             [str(db_path)],
             settings={"num_sql_threads": 1},
+            metadata={
+                "databases": {
+                    "fixtures": {
+                        "tables": {
+                            "facetable": {
+                                "description": "A demo table of places, used to demonstrate facets",
+                                "columns": {"state": "Two letter US state code"},
+                            }
+                        }
+                    }
+                }
+            },
             config={
                 "databases": {
                     "fixtures": {
+                        "tables": {
+                            "facetable": {
+                                "column_types": {"tags": "json"},
+                            }
+                        },
                         "queries": {
                             "neighborhood_search": {
                                 "sql": textwrap.dedent("""
@@ -108,7 +125,7 @@ async def _fetch_live_examples(scoped_classes):
                                 """),
                                 "title": "Search neighborhoods",
                             }
-                        }
+                        },
                     }
                 }
             },

docs/pages.rst

@@ -95,7 +95,7 @@ Use the :ref:`ExecuteWriteView` JSON API to execute writable SQL programmaticall
 Stored query browsers
 ---------------------
 
-The ``/-/queries`` page lists stored queries across every database visible to the current actor. The ``/database-name/-/queries`` page lists stored queries for a single database.
+The ``/-/queries`` page lists stored queries across every database visible to the current actor. The ``/database-name/-/queries`` page lists stored queries for a single database. The JSON versions accept ``?_size=`` (default 50, ``max`` for the :ref:`setting_max_returned_rows` limit) and a ``?_next=`` pagination token.
 
 These pages support search, pagination and filters for read-only or writable queries and private or public queries. Adding a ``.json`` extension to either URL returns the same list as JSON.
 
@@ -118,6 +118,16 @@ Some examples:
 * `../antiquities-act%2Factions_under_antiquities_act <https://fivethirtyeight.datasettes.com/fivethirtyeight/antiquities-act%2Factions_under_antiquities_act>`_ is an interface for exploring the "actions under the antiquities act" data table published by FiveThirtyEight.
 * `../global-power-plants?country_long=United+Kingdom&primary_fuel=Gas <https://datasette.io/global-power-plants/global-power-plants?_facet=primary_fuel&_facet=owner&_facet=country_long&country_long__exact=United+Kingdom&primary_fuel=Gas>`_ is a filtered table page showing every Gas power plant in the United Kingdom. It includes some default facets (configured using `its metadata.json <https://datasette.io/-/metadata>`_) and uses the `datasette-cluster-map <https://github.com/simonw/datasette-cluster-map>`_ plugin to show a map of the results.
 
+.. _TableFragmentView:
+
+Table fragment
+--------------
+
+The ``/<database>/<table>/-/fragment`` endpoint returns the rendered table HTML
+for rows matching the provided filters. It is used by Datasette's row editing
+interface to refresh rows after changes while still respecting custom table
+templates and ``render_cell`` plugin hooks.
+
 .. _RowView:
 
 Row
@@ -159,11 +169,13 @@ Use ``/-/schema.json`` to get the same information as JSON, which looks like thi
 .. code-block:: json
 
     {
+      "ok": true,
       "schemas": [
         {
           "database": "content",
           "schema": "create table posts ..."
         }
+      ]
     }
 
 .. _DatabaseSchemaView:
@@ -171,11 +183,11 @@ Use ``/-/schema.json`` to get the same information as JSON, which looks like thi
 Database schema
 ---------------
 
-Use ``/database-name/-/schema`` to see the complete schema for a specific database. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"database"`` and ``"schema"`` keys.
+Use ``/database-name/-/schema`` to see the complete schema for a specific database. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"ok"``, ``"database"`` and ``"schema"`` keys.
 
 .. _TableSchemaView:
 
 Table schema
 ------------
 
-Use ``/database-name/table-name/-/schema`` to see the schema for a specific table. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"database"``, ``"table"``, and ``"schema"`` keys.
+Use ``/database-name/table-name/-/schema`` to see the schema for a specific table. The ``.md`` and ``.json`` extensions work here too. The JSON returns an object with ``"ok"``, ``"database"``, ``"table"``, and ``"schema"`` keys.

docs/plugin_hooks.rst

@@ -230,6 +230,10 @@ Function that returns an awaitable function that returns a dictionary
 
 Datasette runs Jinja2 in `async mode <https://jinja.palletsprojects.com/en/2.10.x/api/#async-support>`__, which means you can add awaitable functions to the template scope and they will be automatically awaited when they are rendered by the template.
 
+.. warning::
+
+   Be careful not to accidentally define a variable that conflicts with one that Datasette is already using for something else. Check :ref:`the template context documentation <template_context>` to see the variables defined by Datasette core.
+
 Here's an example plugin that adds a ``"user_agent"`` variable to the template context containing the current request's User-Agent header:
 
 .. code-block:: python
@@ -1092,7 +1096,7 @@ Column types are assigned to columns via the :ref:`column_types <table_configura
                 config:
                   format: rgb
 
-Datasette includes three built-in column types: ``url``, ``email``, and ``json``.
+Datasette includes four built-in column types: ``url``, ``email``, ``json``, and ``textarea``. The ``textarea`` type is an editing hint that causes Datasette's insert/edit forms to use a multiline ``<textarea>`` control for that column.
 
 .. _plugin_asgi_wrapper:
 
@@ -1681,6 +1685,8 @@ forbidden(datasette, request, message)
 
 Plugins can use this to customize how Datasette responds when a 403 Forbidden error occurs - usually because a page failed a permission check, see :ref:`authentication_permissions`.
 
+Datasette's default behavior returns the :ref:`standard JSON error format <json_api_errors>` with a 403 status when the request path ends in ``.json`` or the request has an ``Accept: application/json`` or ``Content-Type: application/json`` header; other requests get an HTML error page.
+
 If a plugin hook wishes to react to the error, it should return a :ref:`Response object <internals_response>`.
 
 This example returns a redirect to a ``/-/login`` page:
@@ -1909,7 +1915,79 @@ Action hooks
 
 Action hooks can be used to add items to the action menus that appear at the top of different pages within Datasette. Unlike :ref:`menu_links() <plugin_hook_menu_links>`, actions which are displayed on every page, actions should only be relevant to the page the user is currently viewing.
 
-Each of these hooks should return return a list of ``{"href": "...", "label": "..."}`` menu items, with optional ``"description": "..."`` keys describing each action in more detail.
+Each of these hooks should return a list of menu items, with optional ``"description": "..."`` keys describing each action in more detail.
+
+The most common action item is a link to another page:
+
+.. code-block:: python
+
+    {
+        "href": datasette.urls.path("/-/custom-action"),
+        "label": "Custom action",
+        "description": "Run this action on a separate page.",
+    }
+
+Plugins can also return button actions for JavaScript-backed interactions:
+
+.. code-block:: python
+
+    {
+        "type": "button",
+        "label": "Open custom dialog",
+        "description": "Show a dialog without leaving this page.",
+        "attrs": {
+            "aria-label": "Open custom dialog",
+            "data-plugin-action": "open-custom-dialog",
+        },
+    }
+
+These are rendered as ``<button type="button" class="button-as-link action-menu-button" role="menuitem" tabindex="-1">``. The optional ``attrs`` dictionary is added to the button, and is useful for ``data-*`` attributes that your plugin's JavaScript can use to attach event handlers.
+
+Here is a minimal plugin example that adds a button to a table page and loads JavaScript to handle clicks on that button:
+
+.. code-block:: python
+
+    from datasette import hookimpl
+
+
+    @hookimpl
+    def table_actions(datasette, database, table):
+        return [
+            {
+                "type": "button",
+                "label": "Show table name",
+                "description": "Open a JavaScript-powered plugin action.",
+                "attrs": {
+                    "aria-label": "Show table name",
+                    "data-plugin-action": "show-table-name",
+                    "data-database": database,
+                    "data-table": table,
+                },
+            }
+        ]
+
+
+    @hookimpl
+    def extra_js_urls(datasette):
+        return [
+            datasette.static(
+                "show-table.js", plugin="datasette_show_table"
+            )
+        ]
+
+The ``static/show-table.js`` file in that plugin could look like this:
+
+.. code-block:: javascript
+
+    document.addEventListener("click", (event) => {
+      const button = event.target.closest(
+        "button[data-plugin-action='show-table-name']"
+      );
+      if (!button) {
+        return;
+      }
+      alert(`${button.dataset.database}.${button.dataset.table}`);
+    });
 
 They can alternatively return an ``async def`` awaitable function which, when called, returns a list of those menu items.
 
@@ -2468,6 +2546,10 @@ The default ``SignedTokenHandler`` uses itsdangerous signed tokens (``dstok_`` p
 
         async def verify_token(self, datasette, token):
             # Look up token in database, return actor dict or None
+            # if this handler does not recognize the token. Raise
+            # datasette.TokenInvalid for a token this handler
+            # recognizes but rejects (revoked, expired) - Datasette
+            # will respond with a 401 error.
             ...