Deploy canned queries to latest.datasette.io again with workaround for #2916

Should fix some broken links reported in #2912

.github/workflows/deploy-latest.yml

@@ -14,24 +14,46 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
+    - name: Check deployment prerequisites
+      id: deployment-prerequisites
+      env:
+        GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
+        LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
+      run: |
+        missing=()
+        for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do
+          if [[ -z "${!variable:-}" ]]; then
+            missing+=("$variable")
+          fi
+        done
+        if (( ${#missing[@]} )); then
+          echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}"
+          echo "available=false" >> "$GITHUB_OUTPUT"
+        else
+          echo "available=true" >> "$GITHUB_OUTPUT"
+        fi
     - name: Check out datasette
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/checkout@v7
     - name: Set up Python
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/setup-python@v6
       with:
         python-version: "3.13"
         cache: pip
     - name: Install Python dependencies
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         python -m pip install --upgrade pip
         python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1
+        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
     - name: Run tests
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
     - name: Build fixtures.db and other files needed to deploy the demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |-
         python tests/fixtures.py \
           fixtures.db \
@@ -39,14 +61,18 @@ jobs:
           fixtures-metadata.json \
           plugins \
           --extra-db-filename extra_database.db
+        # Package the config with the plugins, excluding test-only plugin secrets
+        # that reference temporary files outside the deployed container.
+        jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json
     - name: Build docs.db
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
     - name: Set up the alternate-route demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         echo '
         from datasette import hookimpl
@@ -58,6 +84,7 @@ jobs:
         ' > plugins/alternative_route.py
         cp fixtures.db fixtures2.db
     - name: And the counters writable stored query demo
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         cat > plugins/counters.py <<EOF
         from datasette import hookimpl
@@ -97,12 +124,15 @@ jobs:
     #     cat metadata.json
     - id: auth
       name: Authenticate to Google Cloud
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/auth@v3
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}
     - name: Set up Cloud SDK
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/setup-gcloud@v3
     - name: Deploy to Cloud Run
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       env:
         LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
       run: |-
@@ -117,16 +147,16 @@ jobs:
             --plugins-dir=plugins \
             --branch=$GITHUB_SHA \
             --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
+            --extra-options="--config plugins/fixtures-config.json --setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
             --install 'datasette-ephemeral-tables>=0.2.2' \
             --service "datasette-latest$SUFFIX" \
             --secret $LATEST_DATASETTE_SECRET
-    - name: Deploy to docs as well (only for main)
+    - name: Upload latest documentation database to S3 (only for main)
-      if: ${{ github.ref == 'refs/heads/main' }}
+      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
       run: |-
-        # Deploy docs.db to a different service
+        # Keep development documentation separate from the stable release database.
-        datasette publish cloudrun docs.db \
+        s3-credentials put-object datasette-docs latest/docs.db docs.db \
-            --branch=$GITHUB_SHA \
+          --content-type application/octet-stream
-            --version-note=$GITHUB_SHA \
-            --extra-options="--setting template_debug 1" \
-            --service=datasette-docs-latest

.github/workflows/playwright.yml

@@ -2,9 +2,15 @@ name: Playwright
 
 on:
   push:
+    branches:
+    - main
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 permissions:
   contents: read
 

.github/workflows/prettier.yml

@@ -1,6 +1,15 @@
 name: Check JavaScript for conformance with Prettier
 
-on: [push]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read

.github/workflows/publish.yml

@@ -2,7 +2,7 @@ name: Publish Python Package
 
 on:
   release:
-    types: [created]
+    types: [published]
 
 permissions:
   contents: read
@@ -51,6 +51,8 @@ jobs:
     - name: Publish
       uses: pypa/gh-action-pypi-publish@release/v1
 
+  # After the first non-prerelease 1.0 release, disable this job on 0.65.x,
+  # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs.
   deploy_static_docs:
     runs-on: ubuntu-latest
     needs: [deploy]
@@ -66,26 +68,20 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1
+        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
     - name: Build docs.db
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
-    - id: auth
+    - name: Upload stable documentation database to S3
-      name: Authenticate to Google Cloud
+      env:
-      uses: google-github-actions/auth@v2
+        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
-      with:
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
-        credentials_json: ${{ secrets.GCP_SA_KEY }}
-    - name: Set up Cloud SDK
-      uses: google-github-actions/setup-gcloud@v3
-    - name: Deploy stable-docs.datasette.io to Cloud Run
       run: |-
-        gcloud config set run/region us-central1
+        s3-credentials put-object datasette-docs docs.db docs.db \
-        gcloud config set project datasette-222320
+          --content-type application/octet-stream
-        datasette publish cloudrun docs.db \
-            --service=datasette-docs-stable
 
   deploy_docker:
     runs-on: ubuntu-latest

.github/workflows/spellcheck.yml

@@ -1,6 +1,15 @@
 name: Check spelling in documentation
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read

.github/workflows/test-coverage.yml

@@ -1,40 +0,0 @@
-name: Calculate test coverage
-
-on:
-  push:
-    branches:
-      - main
-  pull_request:
-    branches:
-      - main
-permissions:
-  contents: read
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out datasette
-      uses: actions/checkout@v7
-    - name: Set up Python
-      uses: actions/setup-python@v6
-      with:
-        python-version: '3.12'
-        cache: 'pip'
-        cache-dependency-path: '**/pyproject.toml'
-    - name: Install Python dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install . --group dev
-        python -m pip install pytest-cov
-    - name: Run tests
-      run: |-
-        ls -lah
-        cat .coveragerc
-        pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x
-        ls -lah
-    - name: Upload coverage report
-      uses: codecov/codecov-action@v1
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        file: coverage.xml

.github/workflows/test-pyodide.yml

@@ -2,9 +2,15 @@ name: Test in Pyodide with shot-scraper
 
 on:
   push:
+    branches:
+    - main
   pull_request:
   workflow_dispatch:
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 permissions:
   contents: read
 

.github/workflows/test-sqlite-support.yml

@@ -1,6 +1,15 @@
 name: Test SQLite versions
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -12,10 +21,10 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest]
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.13"]
         sqlite-version: [
           #"3", # latest version
-          "3.46",
+          #"3.46",
           #"3.45",
           #"3.27",
           #"3.26",

.github/workflows/test.yml

@@ -1,6 +1,15 @@
 name: Test
 
-on: [push, pull_request]
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -11,16 +20,20 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
+        include:
+        - python-version: "3.14"
+          coverage: true
     steps:
     - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v6
+      uses: actions/setup-python@v7
       with:
         python-version: ${{ matrix.python-version }}
         allow-prereleases: true
         cache: pip
         cache-dependency-path: pyproject.toml
+        check-latest: true
     - name: Build extension for --load-extension test
       run: |-
         (cd tests && gcc ext.c -fPIC -shared -o ext.so)
@@ -28,12 +41,27 @@ jobs:
       run: |
         pip install . --group dev
         pip freeze
+    - name: Install pytest-cov
+      if: ${{ matrix.coverage }}
+      run: pip install pytest-cov
     - name: Run tests
       run: |
-        pytest -n auto -m "not serial"
+        if [ "${{ matrix.coverage }}" = "true" ]; then
-        pytest -m "serial"
+          COV="--cov=datasette --cov-config=.coveragerc"
+          pytest -n auto -m "not serial" $COV --cov-report=
+          pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term
+        else
+          pytest -n auto -m "not serial"
+          pytest -m "serial"
+        fi
         # And the test that exceeds a localhost HTTPS server
         tests/test_datasette_https_server.sh
+    - name: Upload coverage report
+      if: ${{ matrix.coverage }}
+      uses: codecov/codecov-action@v5
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        files: coverage.xml
     - name: Black
       run: |
         black --version

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.0-slim-bullseye as build
+FROM python:3.11-slim-bookworm AS build
 
 # Version of Datasette to install, e.g. 0.55
 #   docker build . -t datasette --build-arg VERSION=0.55

datasette/__init__.py

@@ -1,6 +1,7 @@
 from datasette.permissions import Permission  # noqa
 from datasette.version import __version_info__, __version__  # noqa
 from datasette.events import Event  # noqa
+from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor  # noqa
 from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions  # noqa
 from datasette.utils.asgi import (  # noqa
     Forbidden,

datasette/app.py

@@ -28,7 +28,7 @@ import urllib.parse
 from concurrent import futures
 from pathlib import Path
 
-import httpx
+import httpx2
 from itsdangerous import BadSignature, URLSafeSerializer
 from jinja2 import (
     ChoiceLoader,
@@ -42,6 +42,7 @@ from jinja2.exceptions import TemplateNotFound
 from markupsafe import Markup, escape
 
 from . import stored_queries, write_sql
+from .background_tasks import BackgroundTask, BackgroundTaskSupervisor
 from .column_types import SQLiteType
 from .csrf import CrossOriginProtectionMiddleware
 from .database import Database, QueryInterrupted
@@ -145,6 +146,7 @@ from .views.stored_queries import (
 )
 from .views.table import (
     TableAutocompleteView,
+    TableCountView,
     TableDropView,
     TableFragmentView,
     TableInsertView,
@@ -315,7 +317,7 @@ def _permission_cache_key(actor, action, parent, child):
     actor_key = (
         json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None
     )
-    return (actor_key, action, parent, child)
+    return (actor_key, action.name, parent, action.normalize_child(child))
 
 
 async def favicon(request, send):
@@ -422,6 +424,7 @@ class Datasette:
         default_deny=False,
     ):
         self._startup_invoked = False
+        self._shutdown_invoked = False
         self._closed = False
         assert config_dir is None or isinstance(
             config_dir, Path
@@ -453,8 +456,11 @@ class Datasette:
         self.databases = collections.OrderedDict()
         self.actions = {}  # .invoke_startup() will populate this
         self._column_types = {}  # .invoke_startup() will populate this
+        self._setup_db_done = False
+        self._suppress_background_tasks = False
         try:
             self._refresh_schemas_lock = asyncio.Lock()
+            self._startup_lock = asyncio.Lock()
         except RuntimeError as rex:
             # Workaround for intermittent test failure, see:
             # https://github.com/simonw/datasette/issues/1802
@@ -462,8 +468,10 @@ class Datasette:
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
                 self._refresh_schemas_lock = asyncio.Lock()
+                self._startup_lock = asyncio.Lock()
             else:
                 raise
+        self._background_tasks = BackgroundTaskSupervisor(self)
         self.crossdb = crossdb
         self.nolock = nolock
         if memory or crossdb or not self.files:
@@ -1529,15 +1537,28 @@ class Datasette:
         conn.row_factory = sqlite3.Row
         conn.text_factory = lambda x: str(x, "utf-8", "replace")
         if self.sqlite_extensions and database != INTERNAL_DB_NAME:
+            # Extension loading is only enabled for as long as it takes to
+            # load the configured extensions. Leaving it enabled would let
+            # anyone who can execute SQL call load_extension() themselves.
             conn.enable_load_extension(True)
-            for extension in self.sqlite_extensions:
+            try:
-                # "extension" is either a string path to the extension
+                for extension in self.sqlite_extensions:
-                # or a 2-item tuple that specifies which entrypoint to load.
+                    # "extension" is either a string path to the extension
-                if isinstance(extension, tuple):
+                    # or a 2-item tuple that specifies which entrypoint to load.
-                    path, entrypoint = extension
+                    if isinstance(extension, tuple):
-                    conn.execute("SELECT load_extension(?, ?)", [path, entrypoint])
+                        path, entrypoint = extension
-                else:
+                        if sys.version_info >= (3, 12):
-                    conn.execute("SELECT load_extension(?)", [extension])
+                            conn.load_extension(path, entrypoint=entrypoint)
+                        else:
+                            # Connection.load_extension() only gained the
+                            # entrypoint argument in Python 3.12
+                            conn.execute(
+                                "SELECT load_extension(?, ?)", [path, entrypoint]
+                            )
+                    else:
+                        conn.load_extension(extension)
+            finally:
+                conn.enable_load_extension(False)
         if self.setting("cache_size_kb"):
             conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}")
         # pylint: disable=no-member
@@ -1730,8 +1751,145 @@ class Datasette:
         sql, params = await build_allowed_resources_sql(
             self, actor, action, parent=parent, include_is_private=include_is_private
         )
+        if action == "view-table":
+            sql, params = await self._apply_derived_table_permissions_to_sql(
+                sql,
+                params,
+                actor=actor,
+                parent=parent,
+                include_is_private=include_is_private,
+            )
         return ResourcesSQL(sql, params)
 
+    async def _allowed_derived_table_source(
+        self, database, source, *, actor, dependencies
+    ):
+        """Check an immediate source, denying sources that are themselves derived."""
+        if any(
+            TableResource.normalize_child(table)
+            == TableResource.normalize_child(source)
+            for table in dependencies
+        ):
+            return False
+        # The source has no dependency in this map. Evaluate its own permission
+        # and prerequisites without starting another dependency check.
+        verdicts = await self._allowed_many(
+            actions=["view-table"],
+            resource=TableResource(database, source),
+            actor=actor,
+            check_derived=False,
+        )
+        return verdicts["view-table"]
+
+    async def _apply_derived_table_permissions_to_sql(
+        self,
+        sql,
+        params,
+        *,
+        actor,
+        parent,
+        include_is_private,
+    ):
+        databases = (
+            [(parent, self.databases[parent])]
+            if parent in self.databases
+            else ([] if parent is not None else list(self.databases.items()))
+        )
+        dependency_maps = dict(
+            zip(
+                (name for name, _ in databases),
+                await asyncio.gather(
+                    *(db.derived_table_dependencies() for _, db in databases)
+                ),
+            )
+        )
+        dependencies = [
+            (database_name, child, source)
+            for database_name, dependency_map in dependency_maps.items()
+            for child, source in dependency_map.items()
+        ]
+        if not dependencies:
+            return sql, params
+
+        sources = sorted(
+            {(database_name, source) for database_name, _, source in dependencies}
+        )
+        actor_verdicts = await asyncio.gather(
+            *(
+                self._allowed_derived_table_source(
+                    database_name,
+                    source,
+                    actor=actor,
+                    dependencies=dependency_maps[database_name],
+                )
+                for database_name, source in sources
+            )
+        )
+        actor_allowed = dict(zip(sources, actor_verdicts))
+
+        anonymous_allowed = {}
+        if include_is_private:
+            anonymous_verdicts = await asyncio.gather(
+                *(
+                    self._allowed_derived_table_source(
+                        database_name,
+                        source,
+                        actor=None,
+                        dependencies=dependency_maps[database_name],
+                    )
+                    for database_name, source in sources
+                )
+            )
+            anonymous_allowed = dict(zip(sources, anonymous_verdicts))
+
+        wrapped_params = dict(params)
+        derived_rows = [
+            [
+                database_name,
+                child,
+                int(actor_allowed[(database_name, source)]),
+                *(
+                    [int(anonymous_allowed[(database_name, source)])]
+                    if include_is_private
+                    else []
+                ),
+            ]
+            for database_name, child, source in dependencies
+        ]
+        derived_param = "_datasette_derived_permissions"
+        while derived_param in wrapped_params:
+            derived_param += "_"
+        wrapped_params[derived_param] = json.dumps(derived_rows)
+
+        derived_columns = "parent, child, source_allowed"
+        select_columns = "allowed.parent, allowed.child, allowed.reason"
+        if include_is_private:
+            derived_columns += ", source_anonymous_allowed"
+            select_columns += (
+                ", CASE WHEN derived.source_anonymous_allowed = 0 "
+                "THEN 1 ELSE allowed.is_private END AS is_private"
+            )
+        wrapped_sql = f"""
+WITH derived_permissions({derived_columns}) AS (
+  SELECT
+    json_extract(value, '$[0]'),
+    json_extract(value, '$[1]'),
+    json_extract(value, '$[2]')
+    {", json_extract(value, '$[3]')" if include_is_private else ""}
+  FROM json_each(:{derived_param})
+),
+allowed AS (
+{sql}
+)
+SELECT {select_columns}
+FROM allowed
+LEFT JOIN derived_permissions AS derived
+  ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE
+WHERE COALESCE(derived.source_allowed, 1) = 1
+ORDER BY allowed.parent, allowed.child
+""".strip()
+        return wrapped_sql, wrapped_params
+
     async def allowed_resources(
         self,
         action: str,
@@ -1934,6 +2092,12 @@ class Datasette:
             )
             # {"edit-schema": True, "drop-table": True, "insert-row": False}
         """
+        return await self._allowed_many(
+            actions=actions, resource=resource, actor=actor, check_derived=True
+        )
+
+    async def _allowed_many(self, *, actions, resource, actor, check_derived):
+        """Evaluate permissions, optionally applying the one-hop source policy."""
         from datasette.permissions import (
             _permission_check_cache,
             _skip_permission_checks,
@@ -1971,7 +2135,7 @@ class Datasette:
         to_check = []
         for name in expanded:
             if cache is not None:
-                key = _permission_cache_key(actor, name, parent, child)
+                key = _permission_cache_key(actor, self.actions[name], parent, child)
                 if key in cache:
                     final[name] = cache[key]
                     continue
@@ -1987,6 +2151,28 @@ class Datasette:
                 child=child,
             )
 
+        if (
+            check_derived
+            and "view-table" in to_check
+            and raw.get("view-table")
+            and isinstance(resource, TableResource)
+            and parent in self.databases
+        ):
+            dependencies = await self.databases[parent].derived_table_dependencies()
+            source = next(
+                (
+                    source
+                    for table, source in dependencies.items()
+                    if TableResource.normalize_child(table)
+                    == TableResource.normalize_child(child)
+                ),
+                None,
+            )
+            if source is not None:
+                raw["view-table"] = await self._allowed_derived_table_source(
+                    parent, source, actor=actor, dependencies=dependencies
+                )
+
         def resolve(name):
             # final verdict = own rules AND verdict of also_requires chain
             if name in final:
@@ -2004,7 +2190,9 @@ class Datasette:
         # Cache the freshly computed checks
         if cache is not None:
             for name in to_check:
-                cache[_permission_cache_key(actor, name, parent, child)] = final[name]
+                cache[
+                    _permission_cache_key(actor, self.actions[name], parent, child)
+                ] = final[name]
 
         # Log every check (including cache hits) for the debug page,
         # dependencies before the actions that required them
@@ -2278,6 +2466,21 @@ class Datasette:
         )
         return d
 
+    def _tasks(self):
+        return {
+            "tasks": [
+                {
+                    "name": t.name,
+                    "state": t.state,
+                    "function": t.function,
+                    "started_at": t.started_at,
+                    "exception": repr(t.exception) if t.exception else None,
+                }
+                for t in self._background_tasks.tasks()
+            ],
+            "launched": self._background_tasks.launched,
+        }
+
     def _actor(self, request):
         return {"actor": request.actor}
 
@@ -2446,7 +2649,7 @@ class Datasette:
     ):
         data = {"a": actor}
         if expire_after:
-            expires_at = int(time.time()) + (24 * 60 * 60)
+            expires_at = int(time.time()) + expire_after
             data["e"] = baseconv.base62.encode(expires_at)
         response.set_cookie("ds_actor", self.sign(data, "actor"))
 
@@ -2566,6 +2769,12 @@ class Datasette:
             ),
             r"/-/threads(\.(?P<format>json))?$",
         )
+        add_route(
+            JsonDataView.as_view(
+                self, "tasks.json", self._tasks, permission="permissions-debug"
+            ),
+            r"/-/tasks(\.(?P<format>json))?$",
+        )
         add_route(
             JsonDataView.as_view(
                 self,
@@ -2740,6 +2949,10 @@ class Datasette:
             TableSetColumnTypeView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/set-column-type$",
         )
+        add_route(
+            TableCountView.as_view(self),
+            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/count$",
+        )
         add_route(
             TableFragmentView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/fragment$",
@@ -2803,26 +3016,127 @@ class Datasette:
             raise RowNotFound(db.name, table_name, pk_values)
         return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first())
 
+    async def _startup_sequence(self):
+        """Idempotently run the full startup sequence: table counts for
+        immutable databases, then invoke_startup(). Safe to call more than
+        once and safe to call concurrently - callers block until whichever
+        call got there first has finished.
+
+        This is the single entry point used by both AsgiLifespan (so
+        real deployments finish startup before accepting requests) and
+        AsgiRunOnFirstRequest (the fallback for hosts that never send
+        lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and
+        `datasette serve` (cli.py) calls it too. The fast path below checks
+        both `_startup_invoked` and `_setup_db_done` - not just the former -
+        so that a bare `await ds.invoke_startup()` made by a caller ahead of
+        `_startup_sequence()` (which only sets `_startup_invoked`) can't
+        make this method skip the immutable-database table-count precompute.
+        """
+        if self._startup_invoked and self._setup_db_done:
+            return
+        async with self._startup_lock:
+            if self._startup_invoked and self._setup_db_done:
+                return
+            if not self._setup_db_done:
+                # First time server starts up, calculate table counts for
+                # immutable databases
+                for database in self.databases.values():
+                    if not database.is_mutable:
+                        await database.table_counts(limit=60 * 60 * 1000)
+                self._setup_db_done = True
+            await self.invoke_startup()
+
+    def add_background_task(self, func, name=None) -> BackgroundTask:
+        """Register a piece of supervised background work, typically from
+        a plugin's ``startup`` hook.
+
+        ``func`` must be a coroutine function taking one positional
+        argument, the ``Datasette`` instance - core calls ``func(self)``.
+        Callable any time after ``__init__``: if background tasks haven't
+        launched yet (the common case - most callers are ``startup`` hooks,
+        which run before launch), this buffers the registration until they
+        do; if they've already launched (e.g. called from a request
+        handler after the server is up), the task starts immediately.
+
+        Returns a :class:`~datasette.background_tasks.BackgroundTask`
+        handle (``.name``, ``.state``, ``.task``, ``.exception``,
+        ``.started_at``, ``.function``, ``.cancel()``).
+
+        ``name`` defaults to ``func.__qualname__``; on a name collision a
+        ``-2``, ``-3``, ... suffix is appended, since names are how
+        ``/-/tasks`` and log messages identify work.
+        """
+        return self._background_tasks.add(func, name=name)
+
+    async def start_background_tasks(self):
+        """Run startup (if it hasn't run yet) and launch every registered
+        background task.
+
+        Public entry point for tests, embedders, and headless CLIs (the
+        ``datasette-rss``-style ``fetch --due`` shape) that want supervised
+        background tasks without running a server - equivalent to what
+        happens automatically via ASGI lifespan / the first-request
+        fallback in a served deployment.
+        """
+        await self.invoke_startup()
+        await self._background_tasks.launch_all()
+
+    async def _launch_background_tasks(self):
+        """Idempotently launch every registered background task. Private:
+        this is the entry point wired into the lifecycle trigger lists
+        (the second entry in both ``AsgiLifespan`` and
+        ``AsgiRunOnFirstRequest``'s ``on_startup``, after
+        ``_startup_sequence``) - not something plugins or embedders should
+        call directly; use ``add_background_task`` /
+        ``start_background_tasks`` instead.
+
+        Positioned after ``_startup_sequence`` in both trigger lists so
+        launch always happens once every plugin's ``startup`` hook has had
+        a chance to register work - the ordering guarantee that makes
+        ``add_background_task`` useful. No-ops when
+        ``_suppress_background_tasks`` is set (the ``--get`` CLI path: its
+        one-shot TestClient request flows through the full ASGI stack,
+        including the first-request fallback, but must never launch
+        long-lived background work).
+        """
+        if self._suppress_background_tasks:
+            return
+        await self._background_tasks.launch_all()
+
+    async def invoke_shutdown(self):
+        """Run the graceful teardown sequence: plugin ``shutdown`` hooks,
+        then cancel and drain supervised background tasks, then close
+        every database.
+        """
+        if self._shutdown_invoked:
+            return
+        self._shutdown_invoked = True
+        for hook in pm.hook.shutdown(datasette=self):
+            try:
+                await await_me_maybe(hook)
+            except Exception:
+                logging.getLogger("datasette").exception("shutdown hook failed")
+        await self._background_tasks.cancel_all(grace=5.0)
+        self.close()
+
     def app(self):
         """Returns an ASGI app function that serves the whole of Datasette"""
         routes = self._routes()
 
-        async def setup_db():
-            # First time server starts up, calculate table counts for immutable databases
-            for database in self.databases.values():
-                if not database.is_mutable:
-                    await database.table_counts(limit=60 * 60 * 1000)
-
-        async def _close_on_shutdown():
-            self.close()
-
         asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self)
         if self.setting("trace_debug"):
             asgi = AsgiTracer(asgi)
-        asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown])
+        asgi = AsgiLifespan(
-        asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup])
+            asgi,
+            on_startup=[self._startup_sequence, self._launch_background_tasks],
+            on_shutdown=[self.invoke_shutdown],
+        )
         for wrapper in pm.hook.asgi_wrapper(datasette=self):
             asgi = wrapper(asgi)
+        asgi = AsgiRunOnFirstRequest(
+            asgi,
+            on_startup=[self._startup_sequence, self._launch_background_tasks],
+        )
         return asgi
 
 
@@ -2860,6 +3174,50 @@ class DatasetteRouter:
             receive,
             max_post_body_bytes=self.ds.setting("max_post_body_bytes"),
         )
+        match, view = resolve_routes(self.routes, path)
+        is_static = view is favicon or getattr(view, "_datasette_static", False)
+        original_send = send
+
+        async def send(message):
+            if message["type"] == "http.response.start" and not (
+                is_static and message["status"] in (200, 304)
+            ):
+                # Decide privacy after rendering, including for streaming responses
+                # and error handlers. A public primary resource can still include
+                # private labels, actor navigation, or cookie-dependent content.
+                headers = list(message.get("headers", []))
+                personalized = (
+                    request.actor is not None
+                    or "cookie" in request.headers
+                    or "authorization" in request.headers
+                    or any(key.lower() == b"set-cookie" for key, _ in headers)
+                )
+                if personalized:
+                    headers = [
+                        (key, value)
+                        for key, value in headers
+                        if key.lower() != b"cache-control"
+                    ]
+                    headers.append((b"cache-control", b"private, no-store"))
+
+                # Anonymous responses must not be reused for credentialed requests.
+                # Preserve any additional variation specified by views or plugins.
+                vary = [
+                    part.strip()
+                    for key, value in headers
+                    if key.lower() == b"vary"
+                    for part in value.split(b",")
+                    if part.strip()
+                ]
+                if b"*" not in vary:
+                    for name in (b"Cookie", b"Authorization"):
+                        if name.lower() not in {part.lower() for part in vary}:
+                            vary.append(name)
+                headers = [(k, v) for k, v in headers if k.lower() != b"vary"]
+                headers.append((b"vary", b", ".join(vary)))
+                message = dict(message, headers=headers)
+            await original_send(message)
+
         # Populate request_messages if ds_messages cookie is present
         try:
             request._messages = self.ds.unsign(
@@ -2899,8 +3257,7 @@ class DatasetteRouter:
             return await self.handle_401(request, send, token_error)
         scope_modifications["actor"] = actor or default_actor
         scope = dict(scope, **scope_modifications)
-
+        request.scope = scope
-        match, view = resolve_routes(self.routes, path)
 
         if match is None:
             return await self.handle_404(request, send)
@@ -3215,14 +3572,14 @@ class DatasetteClient:
         with _DatasetteClientContext():
             if skip_permission_checks:
                 with SkipPermissions():
-                    async with httpx.AsyncClient(
+                    async with httpx2.AsyncClient(
-                        transport=httpx.ASGITransport(app=self.app),
+                        transport=httpx2.ASGITransport(app=self.app),
                         cookies=kwargs.pop("cookies", None),
                     ) as client:
                         return await getattr(client, method)(self._fix(path), **kwargs)
             else:
-                async with httpx.AsyncClient(
+                async with httpx2.AsyncClient(
-                    transport=httpx.ASGITransport(app=self.app),
+                    transport=httpx2.ASGITransport(app=self.app),
                     cookies=kwargs.pop("cookies", None),
                 ) as client:
                     return await getattr(client, method)(self._fix(path), **kwargs)
@@ -3269,10 +3626,10 @@ class DatasetteClient:
             method: HTTP method (e.g., "GET", "POST", "PUT")
             path: The path to request
             skip_permission_checks: If True, bypass all permission checks for this request
-            **kwargs: Additional arguments to pass to httpx
+            **kwargs: Additional arguments to pass to httpx2
 
         Returns:
-            httpx.Response: The response from the request
+            httpx2.Response: The response from the request
         """
         from datasette.permissions import SkipPermissions
 
@@ -3281,16 +3638,16 @@ class DatasetteClient:
         with _DatasetteClientContext():
             if skip_permission_checks:
                 with SkipPermissions():
-                    async with httpx.AsyncClient(
+                    async with httpx2.AsyncClient(
-                        transport=httpx.ASGITransport(app=self.app),
+                        transport=httpx2.ASGITransport(app=self.app),
                         cookies=kwargs.pop("cookies", None),
                     ) as client:
                         return await client.request(
                             method, self._fix(path, avoid_path_rewrites), **kwargs
                         )
             else:
-                async with httpx.AsyncClient(
+                async with httpx2.AsyncClient(
-                    transport=httpx.ASGITransport(app=self.app),
+                    transport=httpx2.ASGITransport(app=self.app),
                     cookies=kwargs.pop("cookies", None),
                 ) as client:
                     return await client.request(

datasette/background_tasks.py

@@ -0,0 +1,227 @@
+"""
+Supervised background-task registration for Datasette core.
+
+Plugins that need long-lived background work (a polling loop, a queue
+consumer, a scheduled job runner) register it with
+``datasette.add_background_task(func, name=None)`` - typically from a
+``startup`` plugin hook - instead of fire-and-forgetting their own
+``asyncio.create_task()``. Core owns:
+
+- **references**: every launched ``asyncio.Task`` is kept alive on a
+  :class:`BackgroundTaskSupervisor`, so it can never be silently garbage
+  collected the way an unreferenced ``create_task()`` call can be;
+- **launch timing**: registered work is buffered until
+  :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to
+  happen only after *every* plugin's ``startup`` hook has finished - so
+  a task that depends on another plugin having registered something first
+  doesn't need ``tryfirst=True`` ordering tricks;
+- **crash surfacing**: an unhandled exception in a background task is
+  logged with its full traceback to the ``datasette.background_tasks``
+  logger and recorded on the handle, instead of becoming an "Task
+  exception was never retrieved" warning nobody sees;
+- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels
+  every task still running and waits (with a grace period) for them to
+  actually stop.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import datetime
+import functools
+import logging
+from collections.abc import Awaitable, Callable
+
+logger = logging.getLogger("datasette.background_tasks")
+
+
+def _utcnow_iso() -> str:
+    return datetime.datetime.now(datetime.timezone.utc).isoformat()
+
+
+def _function_path(func: Callable) -> str:
+    """Describe the callable without guessing which plugin registered it."""
+    while isinstance(func, functools.partial):
+        func = func.func
+    if not hasattr(func, "__qualname__"):
+        func = type(func).__call__
+    return f"{func.__module__}.{func.__qualname__}"
+
+
+class BackgroundTask:
+    """A handle to a single piece of supervised background work.
+
+    States: ``registered`` (added but not yet launched) -> ``running`` ->
+    one of ``completed`` (returned cleanly), ``crashed`` (raised an
+    exception other than ``CancelledError`` - see ``.exception``), or
+    ``cancelled`` (``.cancel()`` was called, or it was still running at
+    shutdown).
+    """
+
+    def __init__(
+        self,
+        name: str,
+        func: Callable[[object], Awaitable[None]],
+    ):
+        self.name = name
+        self.state = "registered"
+        self.task: asyncio.Task | None = None
+        self.exception: BaseException | None = None
+        self.started_at: str | None = None
+        self.function = _function_path(func)
+        self._func = func
+        self._supervisor: BackgroundTaskSupervisor | None = None
+
+    def cancel(self) -> None:
+        """Cancel this task.
+
+        If it has already been launched, cancels the underlying
+        ``asyncio.Task`` - its state becomes ``cancelled`` once the
+        cancellation is observed (asynchronously, via the task's done
+        callback). If it has not been launched yet, this is a no-op as
+        far as asyncio is concerned (there's no task to cancel) but it
+        deregisters the handle from its supervisor so it never runs.
+        """
+        if self.task is not None:
+            self.task.cancel()
+        elif self._supervisor is not None:
+            self._supervisor._deregister(self)
+
+    def __repr__(self) -> str:
+        return f"<BackgroundTask name={self.name!r} state={self.state!r}>"
+
+
+class BackgroundTaskSupervisor:
+    """Owns registration and launch of every :class:`BackgroundTask` for a
+    single ``Datasette`` instance.
+
+    Registration (:meth:`add`) is separate from launch
+    (:meth:`launch_all`): plugins register work whenever convenient
+    (typically from a ``startup`` hook, but request handlers can register
+    dynamic per-job work too), and it either sits buffered until
+    :meth:`launch_all` runs, or - if :meth:`launch_all` has already run -
+    starts immediately.
+
+    Strong references to every :class:`BackgroundTask` (and its
+    ``asyncio.Task``) are kept for the life of the instance, by design -
+    that's what makes the enrichments-style "fire-and-forget task gets
+    garbage collected mid-flight" bug impossible here. There is currently
+    no pruning of completed/crashed/cancelled tasks, so a plugin that
+    dynamically registers many short-lived tasks over a long process
+    lifetime (a per-job registration pattern, e.g. one task per queued
+    job) will grow this list without bound. That's an accepted v1
+    trade-off in favour of full introspection (``/-/tasks``); revisit
+    with a pruning or capping policy if unbounded growth is reported in
+    practice.
+    """
+
+    def __init__(self, datasette):
+        self._datasette = datasette
+        self._tasks: list[BackgroundTask] = []
+        self._names = set()
+        self._launched = False
+        self._lock = asyncio.Lock()
+
+    def add(self, func, name=None) -> BackgroundTask:
+        base_name = name or getattr(func, "__qualname__", None) or repr(func)
+        actual_name = self._unique_name(base_name)
+        handle = BackgroundTask(actual_name, func)
+        handle._supervisor = self
+        self._tasks.append(handle)
+        self._names.add(actual_name)
+        if self._launched:
+            self._launch_one(handle)
+        return handle
+
+    def _unique_name(self, base_name: str) -> str:
+        if base_name not in self._names:
+            return base_name
+        n = 2
+        while f"{base_name}-{n}" in self._names:
+            n += 1
+        return f"{base_name}-{n}"
+
+    def _deregister(self, handle: BackgroundTask) -> None:
+        try:
+            self._tasks.remove(handle)
+        except ValueError:
+            pass
+        self._names.discard(handle.name)
+
+    def _launch_one(self, handle: BackgroundTask) -> None:
+        handle.state = "running"
+        handle.started_at = _utcnow_iso()
+        handle.task = asyncio.create_task(
+            handle._func(self._datasette), name=handle.name
+        )
+        handle.task.add_done_callback(functools.partial(_on_task_done, handle))
+
+    async def launch_all(self) -> None:
+        """Launch every currently-registered task that hasn't launched
+        yet. Idempotent and safe to call concurrently: subsequent (or
+        racing) calls are no-ops once the first has set ``self._launched``.
+        """
+        if self._launched:
+            return
+        async with self._lock:
+            if self._launched:
+                return
+            self._launched = True
+            for handle in list(self._tasks):
+                if handle.task is None:
+                    self._launch_one(handle)
+
+    async def cancel_all(self, grace: float = 5.0) -> None:
+        """Cancel every task that isn't already done, then wait up to
+        ``grace`` seconds for them to actually finish. Stragglers still
+        running after that are logged by name (but left to finish or not
+        on their own - this does not forcibly kill them, asyncio has no
+        mechanism for that).
+        """
+        handles_by_task = {
+            handle.task: handle for handle in self._tasks if handle.task is not None
+        }
+        pending = [task for task in handles_by_task if not task.done()]
+        for task in pending:
+            task.cancel()
+        if not pending:
+            return
+        _done, not_done = await asyncio.wait(pending, timeout=grace)
+        if not_done:
+            names = sorted(handles_by_task[task].name for task in not_done)
+            logger.warning(
+                "%d background task(s) did not finish within the %.1fs grace "
+                "period after cancellation: %s",
+                len(names),
+                grace,
+                ", ".join(names),
+            )
+
+    def tasks(self) -> list[BackgroundTask]:
+        """Return every registered :class:`BackgroundTask`, launched or
+        not, in registration order. Used by the ``/-/tasks`` debug
+        endpoint.
+        """
+        return list(self._tasks)
+
+    @property
+    def launched(self) -> bool:
+        """Whether :meth:`launch_all` has run yet - lets ``/-/tasks``
+        distinguish "no tasks registered" from "tasks registered but
+        nothing has armed the launch yet" without reaching for the
+        private ``_launched`` attribute.
+        """
+        return self._launched
+
+
+def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None:
+    if task.cancelled():
+        handle.state = "cancelled"
+        return
+    exc = task.exception()
+    if exc is not None:
+        handle.state = "crashed"
+        handle.exception = exc
+        logger.error("Background task %r crashed", handle.name, exc_info=exc)
+        return
+    handle.state = "completed"

datasette/cli.py

@@ -663,16 +663,6 @@ def serve(
         # Private utility mechanism for writing unit tests
         return ds
 
-    # Run async soundness checks before startup hooks, since invoke_startup
-    # now populates internal tables which requires querying each database
-    run_sync(lambda: check_databases(ds))
-
-    # Run the "startup" plugin hooks
-    try:
-        run_sync(ds.invoke_startup)
-    except StartupError as e:
-        raise click.ClickException(e.args[0])
-
     if headers and not get:
         raise click.ClickException("--headers can only be used with --get")
 
@@ -680,6 +670,19 @@ def serve(
         raise click.ClickException("--token can only be used with --get")
 
     if get:
+        # --get means we don't run Uvicorn at all
+        run_sync(lambda: check_databases(ds))
+
+        try:
+            run_sync(ds.invoke_startup)
+        except StartupError as e:
+            raise click.ClickException(e.args[0])
+
+        # --get never launches background tasks: TestClient's request below
+        # flows through the full ASGI stack, including the
+        # AsgiRunOnFirstRequest fallback, which would otherwise launch them.
+        ds._suppress_background_tasks = True
+
         client = TestClient(ds)
         request_headers = {}
         if token:
@@ -704,34 +707,54 @@ def serve(
         sys.exit(exit_code)
         return
 
-    # Start the server
+    # check_databases, invoke_startup() and the uvicorn server all run on a
-    url = None
+    # single event loop, so that anything a plugin's "startup" hook schedules
-    if root:
+    # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is
-        ds.root_enabled = True
+    # still alive when the server starts handling requests.
-        url = "http://{}:{}{}?token={}".format(
+    async def _serve_async():
-            host, port, ds.urls.path("-/auth-token"), ds._root_token
+        # Populate internal catalog tables before invoke_startup
-        )
+        await check_databases(ds)
-        click.echo(url)
+
-    if open_browser:
+        # Run the full startup sequence (immutable-database table-count
-        if url is None:
+        # precompute + the "startup" plugin hooks) via the same entry point
-            # Figure out most convenient URL - to table, database or homepage
+        # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when
-            path = run_sync(lambda: initial_path_for_datasette(ds))
+        # uvicorn's lifespan.startup fires moments later.
-            url = f"http://{host}:{port}{path}"
+        try:
-        webbrowser.open(url)
+            await ds._startup_sequence()
-    uvicorn_kwargs = {
+        except StartupError as e:
-        "host": host,
+            raise click.ClickException(e.args[0])
-        "port": port,
+
-        "log_level": "info",
+        # Start the server
-        "lifespan": "on",
+        url = None
-        "workers": 1,
+        if root:
-    }
+            ds.root_enabled = True
-    if uds:
+            url = "http://{}:{}{}?token={}".format(
-        uvicorn_kwargs["uds"] = uds
+                host, port, ds.urls.path("-/auth-token"), ds._root_token
-    if ssl_keyfile:
+            )
-        uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile
+            click.echo(url)
-    if ssl_certfile:
+        if open_browser:
-        uvicorn_kwargs["ssl_certfile"] = ssl_certfile
+            if url is None:
-    uvicorn.run(ds.app(), **uvicorn_kwargs)
+                # Figure out most convenient URL - to table, database or homepage
+                path = await initial_path_for_datasette(ds)
+                url = f"http://{host}:{port}{path}"
+            webbrowser.open(url)
+        uvicorn_kwargs = {
+            "host": host,
+            "port": port,
+            "log_level": "info",
+            "lifespan": "on",
+            "workers": 1,
+        }
+        if uds:
+            uvicorn_kwargs["uds"] = uds
+        if ssl_keyfile:
+            uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile
+        if ssl_certfile:
+            uvicorn_kwargs["ssl_certfile"] = ssl_certfile
+        server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs))
+        await server.serve()
+
+    asyncio.run(_serve_async())
 
 
 @cli.command()

datasette/database.py

@@ -29,7 +29,7 @@ from .utils import (
     table_columns,
 )
 from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables
-from .utils.sqlite import sqlite_hidden_table_names
+from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names
 
 connections = threading.local()
 
@@ -85,6 +85,7 @@ class Database:
         self.cached_hash = None
         self.cached_size = None
         self._cached_table_counts = None
+        self._cached_derived_table_dependencies = None
         self._write_thread = None
         self._write_queue = None
         self._closed = False
@@ -246,17 +247,29 @@ class Database:
         return_all=False,
         returning_limit=EXECUTE_WRITE_RETURNING_LIMIT,
         transaction=True,
+        time_limit_ms=2000,
     ):
         self._check_not_closed()
         if returning_limit < 0:
             raise ValueError("returning_limit must be >= 0")
 
-        def _inner(conn):
+        def execute_sql(conn):
             cursor = conn.execute(sql, params or [])
             return ExecuteWriteResult.from_cursor(
                 cursor, return_all=return_all, returning_limit=returning_limit
             )
 
+        def _inner(conn):
+            try:
+                if time_limit_ms is None:
+                    return execute_sql(conn)
+                with sqlite_timelimit(conn, time_limit_ms):
+                    return execute_sql(conn)
+            except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:
+                if e.args == ("interrupted",):
+                    raise QueryInterrupted(e, sql, params)
+                raise
+
         with trace("sql", database=self.name, sql=sql.strip(), params=params):
             results = await self.execute_write_fn(
                 _inner, block=block, request=request, transaction=transaction
@@ -354,6 +367,15 @@ class Database:
                     result = fn(self._write_connection)
             else:
                 result = fn(self._write_connection)
+            if not block:
+                # There is no write thread here, so the write has already
+                # finished. Hand back the same (task_id, reply_future) shape
+                # _send_to_write_thread() returns, with the future already
+                # resolved, so the block=False path below is identical in
+                # both modes.
+                reply_future = asyncio.get_running_loop().create_future()
+                reply_future.set_result(result)
+                result = (uuid.uuid4(), reply_future)
         else:
             result = await self._send_to_write_thread(
                 fn, block=block, transaction=transaction
@@ -425,7 +447,7 @@ class Database:
             )
             self._write_thread.name = f"_execute_writes for database {self.name}"
             self._write_thread.start()
-        task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
+        task_id = uuid.uuid4()
         loop = asyncio.get_running_loop()
         reply_future = loop.create_future()
         self._write_queue.put(
@@ -759,6 +781,17 @@ class Database:
 
         return hidden_tables
 
+    async def derived_table_dependencies(self):
+        """Return implementation tables and the tables they derive from."""
+        schema_version = (await self.execute("PRAGMA schema_version")).first()[0]
+        if (
+            self._cached_derived_table_dependencies is None
+            or self._cached_derived_table_dependencies[0] != schema_version
+        ):
+            dependencies = await self.execute_fn(sqlite_derived_table_dependencies)
+            self._cached_derived_table_dependencies = (schema_version, dependencies)
+        return self._cached_derived_table_dependencies[1]
+
     async def view_names(self):
         results = await self.execute("select name from sqlite_master where type='view'")
         return [r[0] for r in results.rows]

datasette/default_column_types.py

@@ -6,6 +6,17 @@ import markupsafe
 from datasette import hookimpl
 from datasette.column_types import ColumnType, SQLiteType
 
+_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE)
+
+
+def _normalize_http_url(value):
+    if not isinstance(value, str):
+        return None
+    normalized = value.strip()
+    if not _HTTP_URL_RE.fullmatch(normalized):
+        return None
+    return normalized
+
 
 class UrlColumnType(ColumnType):
     name = "url"
@@ -15,7 +26,10 @@ class UrlColumnType(ColumnType):
     async def render_cell(self, value, column, table, database, datasette, request):
         if not value or not isinstance(value, str):
             return None
-        escaped = markupsafe.escape(value.strip())
+        normalized = _normalize_http_url(value)
+        if normalized is None:
+            return markupsafe.escape(value.strip())
+        escaped = markupsafe.escape(normalized)
         return markupsafe.Markup(f'<a href="{escaped}">{escaped}</a>')
 
     async def validate(self, value, datasette):
@@ -23,7 +37,7 @@ class UrlColumnType(ColumnType):
             return None
         if not isinstance(value, str):
             return "URL must be a string"
-        if not re.match(r"^https?://\S+$", value.strip()):
+        if _normalize_http_url(value) is None:
             return "Invalid URL"
         return None
 

datasette/default_permissions/config.py

@@ -92,6 +92,13 @@ class ConfigPermissionProcessor:
             # Tables implicitly reference their parent databases
             self.restricted_databases.update(db for db, _ in self.restricted_tables)
 
+        # Resolve identity keys once per action, rather than scanning the
+        # restriction allowlist for every configured table's allow block.
+        self.restricted_table_keys = {
+            (db, self.action_obj.normalize_child(table) if self.action_obj else table)
+            for db, table in self.restricted_tables
+        }
+
     def evaluate_allow_block(self, allow_block: Any) -> bool | None:
         """Evaluate an allow block against the current actor."""
         if allow_block is None:
@@ -125,8 +132,10 @@ class ConfigPermissionProcessor:
         if parent:
             table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {})
             if child:
-                table_actions = table_restrictions.get(child, [])
+                child_key = (
-                if self.action_checks.intersection(table_actions):
+                    self.action_obj.normalize_child(child) if self.action_obj else child
+                )
+                if (parent, child_key) in self.restricted_table_keys:
                     return True
             else:
                 # Parent query should proceed if any child in this database is allowlisted

datasette/default_permissions/restrictions.py

@@ -185,11 +185,15 @@ def restrictions_allow_action(
     # Check table/resource level
     if resource is not None and not isinstance(resource, str) and len(resource) == 2:
         database, table = resource
-        table_allowed = restrictions.get("r", {}).get(database, {}).get(table)
+        action_obj = datasette.actions.get(action)
-        if table_allowed is not None:
+        normalize = action_obj.normalize_child if action_obj else lambda name: name
-            assert isinstance(table_allowed, list)
+        for table_name, table_allowed in (
-            if to_check.intersection(table_allowed):
+            restrictions.get("r", {}).get(database, {}).items()
-                return True
+        ):
+            if normalize(table_name) == normalize(table):
+                assert isinstance(table_allowed, list)
+                if to_check.intersection(table_allowed):
+                    return True
 
     # This action is not explicitly allowed, so reject it
     return False

datasette/default_permissions/sqlite_statistics.py

@@ -0,0 +1,25 @@
+"""Default table-access policy for SQLite optimizer statistics."""
+
+import json
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+
+@hookimpl
+def permission_resources_sql(action):
+    if action != "view-table":
+        return None
+    return PermissionSQL(
+        sql="""
+            SELECT database_name AS parent, value AS child, 0 AS allow,
+                'SQLite statistics tables are denied by default' AS reason
+            FROM catalog_databases
+            CROSS JOIN json_each(:sqlite_statistics_names)
+        """,
+        params={
+            "sqlite_statistics_names": json.dumps(
+                ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"]
+            )
+        },
+    )

datasette/filters.py

@@ -2,7 +2,7 @@ import json
 from typing import ClassVar
 
 from datasette import hookimpl
-from datasette.resources import DatabaseResource
+from datasette.resources import DatabaseResource, TableResource
 from datasette.utils.asgi import BadRequest
 from datasette.views.base import DatasetteError
 
@@ -51,13 +51,20 @@ def search_filters(request, database, table, datasette):
         human_descriptions = []
         extra_context = {}
 
-        # Figure out which fts_table to use
+        # Figure out which trusted fts_table to use. Query string parameters can
+        # repeat this mapping (for backwards compatibility), but must not select
+        # a different table or primary key.
         table_metadata = await datasette.table_config(database, table)
         db = datasette.get_database(database)
-        fts_table = request.args.get("_fts_table")
+        fts_table = table_metadata.get("fts_table")
-        fts_table = fts_table or table_metadata.get("fts_table")
         fts_table = fts_table or await db.fts_table(table)
-        fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid"))
+        fts_pk = table_metadata.get("fts_pk", "rowid")
+        requested_fts_table = request.args.get("_fts_table")
+        requested_fts_pk = request.args.get("_fts_pk")
+        if (requested_fts_table and requested_fts_table != fts_table) or (
+            requested_fts_pk and requested_fts_pk != fts_pk
+        ):
+            raise BadRequest("Invalid _fts_table or _fts_pk")
         search_args = {
             key: request.args[key]
             for key in request.args
@@ -75,6 +82,11 @@ def search_filters(request, database, table, datasette):
         extra_context["supports_search"] = bool(fts_table)
 
         if fts_table and search_args:
+            await datasette.ensure_permission(
+                action="view-table",
+                resource=TableResource(database=database, table=fts_table),
+                actor=request.actor,
+            )
             if "_search" in search_args:
                 # Simple ?_search=xxx
                 search = search_args["_search"]
@@ -135,6 +147,11 @@ def through_filters(request, database, table, datasette):
                 through_table = through_data["table"]
                 other_column = through_data["column"]
                 value = through_data["value"]
+                await datasette.ensure_permission(
+                    action="view-table",
+                    resource=TableResource(database=database, table=through_table),
+                    actor=request.actor,
+                )
                 db = datasette.get_database(database)
                 outgoing_foreign_keys = await db.foreign_keys_for_table(through_table)
                 fk_to_us = next(

datasette/hookspecs.py

@@ -9,6 +9,11 @@ def startup(datasette):
     """Fires directly after Datasette first starts running"""
 
 
+@hookspec
+def shutdown(datasette):
+    """Called once when the Datasette server is shutting down"""
+
+
 @hookspec
 def asgi_wrapper(datasette):
     """Returns an ASGI middleware callable to wrap our ASGI application with"""

datasette/permissions.py

@@ -3,6 +3,10 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Any, NamedTuple
 
+_SQLITE_IDENTIFIER_CASE = str.maketrans(
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
+)
+
 # Context variable to track when permission checks should be skipped
 _skip_permission_checks = contextvars.ContextVar(
     "skip_permission_checks", default=False
@@ -49,6 +53,15 @@ class Resource(ABC):
     # Class-level metadata (subclasses must define these)
     name: str = None  # e.g., "table", "database", "model"
     parent_class: type["Resource"] | None = None  # e.g., DatabaseResource for tables
+    case_insensitive_child: bool = False
+
+    @classmethod
+    def normalize_child(cls, child: str | None) -> str | None:
+        """Return a comparison key without changing the resource's display name."""
+        if cls.case_insensitive_child and child is not None:
+            # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold.
+            return child.translate(_SQLITE_IDENTIFIER_CASE)
+        return child
 
     # Instance-level optional extra attributes
     reasons: list[str] | None = None
@@ -146,6 +159,11 @@ class Action:
     resource_class: type[Resource] | None = None
     also_requires: str | None = None  # Optional action name that must also be allowed
 
+    def normalize_child(self, child: str | None) -> str | None:
+        if self.resource_class is None:
+            return child
+        return self.resource_class.normalize_child(child)
+
     @property
     def takes_parent(self) -> bool:
         """

datasette/plugins.py

@@ -18,6 +18,7 @@ DEFAULT_PLUGINS = (
     "datasette.actor_auth_cookie",
     "datasette.default_permissions",
     "datasette.default_permissions.tokens",
+    "datasette.default_permissions.sqlite_statistics",
     "datasette.default_actions",
     "datasette.default_column_types",
     "datasette.default_magic_parameters",

datasette/resources.py

@@ -25,6 +25,7 @@ class TableResource(Resource):
 
     name = "table"
     parent_class = DatabaseResource
+    case_insensitive_child = True
 
     def __init__(self, database: str, table: str):
         super().__init__(parent=database, child=table)

datasette/static/app.css

@@ -216,6 +216,49 @@ a:active  {
 	text-decoration: underline;
 }
 
+.table-summary .count-all ~ .table-summary-description {
+    margin-left: 0.5rem;
+}
+
+.table-summary .count-error:not(:empty) {
+    display: block;
+    margin-top: 0.25rem;
+    font-size: 0.875rem;
+    font-weight: 400;
+    line-height: 1.5;
+}
+
+button.count-all {
+    background: none;
+    border: none;
+    padding: 3px 0;
+    margin-left: 0.25rem;
+    color: #276890;
+    font-family: inherit;
+    font-size: 0.8125rem;
+    font-weight: 400;
+    line-height: 1.5;
+    cursor: pointer;
+}
+
+button.count-all:hover,
+button.count-all:focus-visible {
+    text-decoration: underline;
+}
+
+button.count-all:disabled {
+    color: #596478;
+    cursor: wait;
+}
+
+@media (pointer: coarse) {
+    button.count-all {
+        min-height: 44px;
+        padding-left: 7px;
+        padding-right: 7px;
+    }
+}
+
 button.button-as-link {
     background: none;
     border: none;

datasette/static/column-chooser.js

@@ -472,11 +472,13 @@ class ColumnChooser extends HTMLElement {
           <span class="drag-item-check">
             <input type="checkbox" ${this._checked.has(col) ? "checked" : ""}>
           </span>
-          <span class="drag-item-label">${col}</span>
+          <span class="drag-item-label"></span>
         </label>
         <div class="drop-indicator"></div>
       `;
 
+      li.querySelector(".drag-item-label").textContent = col;
+
       li.querySelector("input").addEventListener("change", (e) => {
         e.target.checked ? this._checked.add(col) : this._checked.delete(col);
         this._updateCounts();

datasette/static/json-format-highlight-1.0.1.js

@@ -1,56 +0,0 @@
-/*
-https://github.com/luyilin/json-format-highlight
-From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js
-MIT Licensed
-*/
-(function (global, factory) {
-  typeof exports === "object" && typeof module !== "undefined"
-    ? (module.exports = factory())
-    : typeof define === "function" && define.amd
-      ? define(factory)
-      : (global.jsonFormatHighlight = factory());
-})(this, function () {
-  "use strict";
-
-  var defaultColors = {
-    keyColor: "dimgray",
-    numberColor: "lightskyblue",
-    stringColor: "lightcoral",
-    trueColor: "lightseagreen",
-    falseColor: "#f66578",
-    nullColor: "cornflowerblue",
-  };
-
-  function index(json, colorOptions) {
-    if (colorOptions === void 0) colorOptions = {};
-
-    if (!json) {
-      return;
-    }
-    if (typeof json !== "string") {
-      json = JSON.stringify(json, null, 2);
-    }
-    var colors = Object.assign({}, defaultColors, colorOptions);
-    json = json.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
-    return json.replace(
-      /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g,
-      function (match) {
-        var color = colors.numberColor;
-        if (/^"/.test(match)) {
-          color = /:$/.test(match) ? colors.keyColor : colors.stringColor;
-        } else {
-          color = /true/.test(match)
-            ? colors.trueColor
-            : /false/.test(match)
-              ? colors.falseColor
-              : /null/.test(match)
-                ? colors.nullColor
-                : color;
-        }
-        return '<span style="color: ' + color + '">' + match + "</span>";
-      },
-    );
-  }
-
-  return index;
-});

datasette/static/table.js

@@ -860,10 +860,45 @@ function openColumnChooser() {
   });
 }
 
+function initCountAll() {
+  var button = document.querySelector(".count-all");
+  if (!button) {
+    return;
+  }
+  button.addEventListener("click", async function () {
+    var count = document.querySelector(".table-count");
+    var error = document.querySelector(".count-error");
+    button.disabled = true;
+    button.textContent = "Counting…";
+    error.textContent = "";
+    try {
+      var response = await fetch(button.dataset.countUrl + location.search, {
+        method: "POST",
+        headers: {
+          Accept: "application/json",
+        },
+      });
+      var data = await response.json();
+      if (!response.ok || !data.ok) {
+        throw new Error((data.errors || ["Count failed"]).join(" "));
+      }
+      count.textContent =
+        data.count.toLocaleString("en-US") +
+        (data.count === 1 ? " row" : " rows");
+      button.remove();
+    } catch (ex) {
+      error.textContent = ex.message || "Count failed";
+      button.disabled = false;
+      button.textContent = "count all";
+    }
+  });
+}
+
 // Ensures Table UI is initialized only after the Manager is ready.
 document.addEventListener("datasette_init", function (evt) {
   const { detail: manager } = evt;
 
+  initCountAll();
   initializeColumnActions(manager);
 
   // Main table

datasette/templates/api_explorer.html

@@ -3,7 +3,6 @@
 {% block title %}API Explorer{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% endblock %}
 
 {% block content %}
@@ -126,7 +125,7 @@ getForm.addEventListener("submit", (ev) => {
     document.getElementById('response-status').textContent = response.status;
     return response.json();
   }).then((data) => {
-    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
+    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
     errorList.style.display = 'none';
   }).catch((error) => {
     alert(error);
@@ -174,7 +173,7 @@ postForm.addEventListener("submit", (ev) => {
     } else {
       errorList.style.display = 'none';
     }
-    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
+    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
     output.style.display = 'block';
   }).catch(err => {
     alert("Error: " + err);

datasette/templates/debug_allowed.html

@@ -3,7 +3,6 @@
 {% block title %}Allowed Resources{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -198,7 +197,7 @@ function displayResults(data) {
     }
 
     // Update raw JSON
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayError(data) {
@@ -208,7 +207,7 @@ function displayError(data) {
 
     resultsContent.innerHTML = `<div class="error-message">Error: ${escapeHtml(data.error || 'Unknown error')}</div>`;
 
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 // Disable child input if parent is empty

datasette/templates/debug_check.html

@@ -3,7 +3,6 @@
 {% block title %}Explain a permission decision{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 <style>
@@ -238,7 +237,7 @@ function displayResult(data) {
     displayRules(data.explanation);
     displayRestrictions(data.explanation.restrictions);
     displayRequirements(data.explanation.required_actions);
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayRules(explanation) {
@@ -298,7 +297,7 @@ function displayError(data) {
     document.getElementById('matching-rules').innerHTML = '';
     document.getElementById('restrictions-section').style.display = 'none';
     document.getElementById('requirements-section').style.display = 'none';
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 form.addEventListener('submit', event => {

datasette/templates/debug_rules.html

@@ -3,7 +3,6 @@
 {% block title %}Permission Rules{% endblock %}
 
 {% block extra_head %}
-<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -185,7 +184,7 @@ function displayResults(data) {
     }
 
     // Update raw JSON
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 function displayError(data) {
@@ -195,7 +194,7 @@ function displayError(data) {
 
     resultsContent.innerHTML = `<div class="error-message">Error: ${escapeHtml(data.error || 'Unknown error')}</div>`;
 
-    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
+    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
 }
 
 </script>

datasette/templates/table.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 
-{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
+{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
 
 {% block extra_head %}
 {{- super() -}}
@@ -47,11 +47,12 @@
 {% endif %}
 
 {% if count or human_description_en %}
-    <h3>
+    <h3 class="table-summary">
-        {% if count_truncated %}&gt;{{ "{:,}".format(count - 1) }} rows
+        {% if count_truncated %}<span class="table-count" aria-live="polite">{{ "{:,}".format(count - 1) }}+ rows</span>
-        {% if allow_execute_sql and query.sql %} <a class="count-sql" style="font-size: 0.8em;" href="{{ urls.database_query(database, count_sql) }}">count all</a>{% endif %}
+        <button type="button" class="count-all" data-count-url="{{ urls.table(database, table) }}/-/count">count all</button>
+        <span class="count-error" role="alert"></span>
         {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}
-        {% if human_description_en %}{{ human_description_en }}{% endif %}
+        {% if human_description_en %}<span class="table-summary-description">{{ human_description_en }}</span>{% endif %}
     </h3>
 {% endif %}
 

datasette/utils/__init__.py

@@ -820,7 +820,8 @@ def detect_spatialite(conn):
 
 def detect_fts(conn, table):
     """Detect if table has a corresponding FTS virtual table and return it"""
-    rows = conn.execute(detect_fts_sql(table)).fetchall()
+    sql, params = detect_fts_sql(table)
+    rows = conn.execute(sql, params).fetchall()
     if len(rows) == 0:
         return None
     else:
@@ -828,18 +829,26 @@ def detect_fts(conn, table):
 
 
 def detect_fts_sql(table):
-    return r"""
+    escaped_table = table.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
-        select name from sqlite_master
+    return (
-            where rootpage = 0
+        r"""
-            and (
+            select name from sqlite_master
-                sql like '%VIRTUAL TABLE%USING FTS%content="{table}"%'
+                where rootpage = 0
-                or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%'
+                and (
-                or (
+                    sql like :fts_double_quoted escape char(92)
-                    tbl_name = "{table}"
+                    or sql like :fts_bracket_quoted escape char(92)
-                    and sql like '%VIRTUAL TABLE%USING FTS%'
+                    or (
+                        tbl_name = :table
+                        and sql like '%VIRTUAL TABLE%USING FTS%'
+                    )
                 )
-            )
+        """,
-    """.format(table=table.replace("'", "''"))
+        {
+            "fts_double_quoted": f'%VIRTUAL TABLE%USING FTS%content="{escaped_table}"%',
+            "fts_bracket_quoted": f"%VIRTUAL TABLE%USING FTS%content=[{escaped_table}]%",
+            "table": table,
+        },
+    )
 
 
 def detect_json1(conn=None):
@@ -1557,7 +1566,13 @@ async def row_sql_params_pks(db, table, pk_values):
     if use_rowid:
         select = "rowid, *"
         pks = ["rowid"]
-    wheres = [f'"{pk}"=:p{i}' for i, pk in enumerate(pks)]
+    wheres = []
+    for i, pk in enumerate(pks):
+        escaped_pk = escape_sqlite(pk)
+        # Preserve the historic always-quoted SQL exposed by _extra=query
+        if escaped_pk == pk:
+            escaped_pk = f'"{pk}"'
+        wheres.append(f"{escaped_pk}=:p{i}")
     sql = f"select {select} from {escape_sqlite(table)} where {' AND '.join(wheres)}"
     params = {}
     for i, pk_value in enumerate(pk_values):
@@ -1729,7 +1744,7 @@ def redact_keys(original: dict, key_patterns: Iterable) -> dict:
             return {
                 k: (
                     redact(v)
-                    if not any(pattern in k for pattern in key_patterns)
+                    if not any(pattern in k.casefold() for pattern in key_patterns)
                     else "***"
                 )
                 for k, v in data.items()

datasette/utils/actions_sql.py

@@ -29,6 +29,15 @@ from datasette.utils.permissions import gather_permission_sql_from_hooks
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
+    from datasette.permissions import Action
+
+
+def _child_collation(action: "Action") -> str:
+    """Match resource identity without changing the spelling returned by SQL."""
+    resource_class = action.resource_class
+    if resource_class is not None and resource_class.case_insensitive_child:
+        return "NOCASE"
+    return "BINARY"
 
 
 async def build_allowed_resources_sql(
@@ -149,6 +158,7 @@ async def _build_single_action_sql(
         raise ValueError(f"Unknown action: {action}")
 
     # Get base resources SQL from the resource class
+    child_collation = _child_collation(action_obj)
     base_resources_sql = await action_obj.resource_class.resources_sql(
         datasette, actor=actor
     )
@@ -185,7 +195,7 @@ async def _build_single_action_sql(
         if permission_sql.sql is None:
             continue
         rule_sqls.append(f"""
-            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
+            SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
             """.strip())
@@ -299,9 +309,9 @@ async def _build_single_action_sql(
         query_parts.extend(
             ["anon_child_agg AS ("]
             + _anon_agg(
-                "parent, child,",
+                f"parent, child COLLATE {child_collation} AS child,",
                 "parent IS NOT NULL AND child IS NOT NULL",
-                "parent, child",
+                f"parent, child COLLATE {child_collation}",
             )
             + ["),", "anon_parent_agg AS ("]
             + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent")
@@ -382,7 +392,8 @@ async def _build_single_action_sql(
         # Wrap each restriction_sql in a subquery to avoid operator precedence issues
         # with UNION ALL inside the restriction SQL statements
         restriction_intersect = "\nINTERSECT\n".join(
-            f"SELECT * FROM ({sql})" for sql in restriction_sqls
+            f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})"
+            for sql in restriction_sqls
         )
         # Decompose by NULL-pattern so the final filter can use pure-equality
         # EXISTS lookups (satisfiable via automatic indexes) instead of a
@@ -480,6 +491,7 @@ async def build_permission_rules_sql(
     union_parts = []
     all_params = {}
     restriction_sqls = []
+    child_collation = _child_collation(action_obj)
 
     for permission_sql in permission_sqls:
         all_params.update(permission_sql.params or {})
@@ -493,7 +505,7 @@ async def build_permission_rules_sql(
             continue
 
         union_parts.append(f"""
-            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
+            SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
             """.strip())
@@ -564,6 +576,7 @@ async def check_permissions_for_actions(
     verdicts = {}
 
     for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)):
+        child_collation = _child_collation(datasette.actions[action])
         prefix = f"a{i}_"
         rule_parts = []
         restriction_parts = []
@@ -589,7 +602,7 @@ async def check_permissions_for_actions(
             if sql is None:
                 continue
             rule_parts.append(
-                f"SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
+                f"SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
             )
 
         if not rule_parts:
@@ -623,7 +636,8 @@ async def check_permissions_for_actions(
         if restriction_parts:
             # Database-level restrictions (parent, NULL) match all children
             restriction_intersect = "\nINTERSECT\n".join(
-                f"SELECT * FROM ({sql})" for sql in restriction_parts
+                f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})"
+                for sql in restriction_parts
             )
             ctes.append(f"a{i}_restriction AS (\n{restriction_intersect}\n)")
             verdict_sql = f"""({verdict_sql}) AND EXISTS (
@@ -770,6 +784,7 @@ async def _explain_single_action(
     db = datasette.get_internal_database()
     matched_rules = []
     restrictions = []
+    child_collation = _child_collation(datasette.actions[action])
 
     for permission_sql in permission_sqls:
         params = dict(permission_sql.params or {})
@@ -784,7 +799,7 @@ async def _explain_single_action(
                 SELECT parent, child, allow, reason
                 FROM ({permission_sql.sql}) AS permission_rules
                 WHERE (parent IS NULL OR parent = :{parent_param})
-                  AND (child IS NULL OR child = :{child_param})
+                  AND (child IS NULL OR child COLLATE {child_collation} = :{child_param})
                 """,
                 params,
             )
@@ -811,7 +826,7 @@ async def _explain_single_action(
                     SELECT EXISTS(
                         SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules
                         WHERE (parent IS NULL OR parent = :{parent_param})
-                          AND (child IS NULL OR child = :{child_param})
+                          AND (child IS NULL OR child COLLATE {child_collation} = :{child_param})
                     ) AS resource_is_in_allowlist
                     """,
                     params,

datasette/utils/asgi.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import re
 from http.cookies import Morsel, SimpleCookie
@@ -300,12 +301,24 @@ class AsgiLifespan:
             while True:
                 message = await receive()
                 if message["type"] == "lifespan.startup":
-                    for fn in self.on_startup:
+                    try:
-                        await fn()
+                        for fn in self.on_startup:
+                            await fn()
+                    except Exception as e:  # noqa: BLE001
+                        await send(
+                            {"type": "lifespan.startup.failed", "message": str(e)}
+                        )
+                        return
                     await send({"type": "lifespan.startup.complete"})
                 elif message["type"] == "lifespan.shutdown":
-                    for fn in self.on_shutdown:
+                    try:
-                        await fn()
+                        for fn in self.on_shutdown:
+                            await fn()
+                    except Exception as e:  # noqa: BLE001
+                        await send(
+                            {"type": "lifespan.shutdown.failed", "message": str(e)}
+                        )
+                        return
                     await send({"type": "lifespan.shutdown.complete"})
                     return
         else:
@@ -485,6 +498,8 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None):
             await asgi_send_html(send, "404: File not found", 404)
             return
 
+    # Only the actual static-file handler can bypass dynamic response privacy.
+    inner_static._datasette_static = True
     return inner_static
 
 
@@ -624,10 +639,23 @@ class AsgiRunOnFirstRequest:
         self.asgi = asgi
         self.on_startup = on_startup
         self._started = False
+        # Guards against concurrent early requests interleaving with startup:
+        # without this, several requests could all observe `_started is
+        # False` and proceed before any of them finish running the hooks.
+        self._lock = asyncio.Lock()
 
     async def __call__(self, scope, receive, send):
-        if not self._started:
+        # Leave "lifespan" scope events alone - this shim only exists as a
-            self._started = True
+        # fallback for hosts that never send them. It wraps AsgiLifespan, so
-            for hook in self.on_startup:
+        # if it ran on_startup here too, a startup exception would escape
-                await hook()
+        # before AsgiLifespan's own try/except got a chance to turn it into
+        # a lifespan.startup.failed message.
+        if scope["type"] != "lifespan" and not self._started:
+            async with self._lock:
+                # Re-check: another request may have finished startup while
+                # we were waiting for the lock.
+                if not self._started:
+                    for hook in self.on_startup:
+                        await hook()
+                    self._started = True
         return await self.asgi(scope, receive, send)

datasette/utils/sql_analysis.py

@@ -1,6 +1,8 @@
+import sys
 from dataclasses import dataclass
 from typing import Literal
 
+from datasette.utils import escape_sqlite
 from datasette.utils.sqlite import SQLiteTableType, sqlite3, sqlite_table_type
 
 SQLOperation = Literal[
@@ -195,6 +197,16 @@ def _allow_authorizer_action(*args):
     return sqlite3.SQLITE_OK
 
 
+def _disable_authorizer(conn):
+    # Python 3.11 added support for unregistering an authorizer using None.
+    # On Python 3.10, None is installed as the callback instead, and the next
+    # statement fails with "not authorized" when sqlite3 tries to call it.
+    if sys.version_info >= (3, 11):
+        conn.set_authorizer(None)
+    else:
+        conn.set_authorizer(_allow_authorizer_action)
+
+
 def analyze_sql_tables(
     conn,
     sql: str,
@@ -208,7 +220,9 @@ def analyze_sql_tables(
 
     This function is synchronous and connection-based. It temporarily installs a
     SQLite authorizer, prepares ``EXPLAIN <sql>``, and returns the operation
-    callbacks observed while SQLite compiles the statement.
+    callbacks observed while SQLite compiles the statement. ``CREATE VIEW`` is
+    additionally executed inside a rolled-back savepoint so its source-table reads
+    can be discovered by analyzing a query against the temporary view.
     """
     operations: dict[OperationKey, set[str]] = {}
 
@@ -481,7 +495,7 @@ def analyze_sql_tables(
                         conn, key.table, schema=key.sqlite_schema
                     )
     finally:
-        conn.set_authorizer(None)
+        _disable_authorizer(conn)
 
     has_schema_operation = any(
         key.target_type in {"table", "index", "view", "trigger", "virtual-table"}
@@ -532,7 +546,7 @@ def analyze_sql_tables(
             return None
         return table_kind_cache[(key.sqlite_schema, key.table)]
 
-    return SQLAnalysis(
+    analysis = SQLAnalysis(
         operations=tuple(
             Operation(
                 operation=key.operation,
@@ -549,3 +563,58 @@ def analyze_sql_tables(
             for key, columns in operations.items()
         )
     )
+
+    # SQLite does not resolve the SELECT body of a view when preparing CREATE
+    # VIEW, so its authorizer does not report reads from the view's source
+    # tables. Temporarily create the view, analyze a query against it (which
+    # does resolve the body), then roll the schema change back. Database-level
+    # callers use an isolated writable connection for this analysis.
+    create_view_operations = tuple(
+        operation
+        for operation in analysis.operations
+        if operation.operation == "create" and operation.target_type == "view"
+    )
+    if not create_view_operations:
+        return analysis
+
+    savepoint = "datasette_analyze_create_view"
+    conn.execute(f"SAVEPOINT {savepoint}")
+    try:
+        conn.execute(sql, params if params is not None else {})
+        dependency_reads = []
+        for view_operation in create_view_operations:
+            if view_operation.sqlite_schema is None or view_operation.table is None:
+                raise sqlite3.OperationalError(
+                    "Could not determine the created view name"
+                )
+            quoted_schema = escape_sqlite(view_operation.sqlite_schema)
+            quoted_view = escape_sqlite(view_operation.table)
+            qualified_view = f"{quoted_schema}.{quoted_view}"
+            view_analysis = analyze_sql_tables(
+                conn,
+                f"SELECT * FROM {qualified_view}",
+                database_name=database_name,
+                schema_to_database=schema_to_database,
+            )
+            dependency_reads.extend(
+                operation
+                for operation in view_analysis.operations
+                if operation.operation == "read"
+                and not (
+                    operation.sqlite_schema == view_operation.sqlite_schema
+                    and operation.table == view_operation.table
+                )
+            )
+    finally:
+        conn.execute(f"ROLLBACK TO {savepoint}")
+        conn.execute(f"RELEASE {savepoint}")
+
+    existing_operations = set(analysis.operations)
+    return SQLAnalysis(
+        operations=analysis.operations
+        + tuple(
+            operation
+            for operation in dependency_reads
+            if operation not in existing_operations
+        )
+    )

datasette/utils/sqlite.py

@@ -15,8 +15,17 @@ if hasattr(sqlite3, "enable_callback_tracebacks"):
 _cached_sqlite_version = None
 _cached_supports_returning = None
 SQLiteTableType = Literal["table", "view", "virtual", "shadow"]
+_SQLITE_IDENTIFIER_RE = (
+    r"""(?:"(?:[^"]|"")*"|'(?:[^']|'')*'|`(?:[^`]|``)*`|\[[^\]]*\]|[^\s.()'"`\[\]]+)"""
+)
 _VIRTUAL_TABLE_MODULE_RE = re.compile(
-    r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)",
+    r"^\s*CREATE\s+VIRTUAL\s+TABLE\b\s*(?:IF\s+NOT\s+EXISTS\s+)?"
+    + _SQLITE_IDENTIFIER_RE
+    + r"(?:\s*\.\s*"
+    + _SQLITE_IDENTIFIER_RE
+    + r")?\s*\bUSING\b\s*("
+    + _SQLITE_IDENTIFIER_RE
+    + r")",
     re.IGNORECASE | re.DOTALL,
 )
 _VIRTUAL_TABLE_SHADOW_SUFFIXES = {
@@ -83,19 +92,53 @@ def sqlite_table_type(
 ) -> SQLiteTableType | None:
     if supports_table_list():
         try:
-            query = "select type from pragma_table_list where name = ?"
+            # Use the "PRAGMA table_list" statement form rather than the
-            params: tuple[str, ...] = (table,)
+            # pragma_table_list(...) table-valued function. The
+            # table-valued function is resolved like an ordinary relation
+            # name, so an attacker-created table or view literally named
+            # "pragma_table_list" can shadow it and spoof the reported
+            # type (e.g. claiming a virtual table is an ordinary table).
+            # The PRAGMA statement form is a distinct piece of SQL syntax
+            # that always invokes SQLite's built-in pragma, so it cannot
+            # be shadowed by a user-created relation.
             if schema is not None:
-                query += " and schema = ?"
+                query = f"PRAGMA {_quote_identifier(schema)}.table_list"
-                params = (table, schema)
+            else:
-            row = conn.execute(query, params).fetchone()
+                query = "PRAGMA table_list"
-            if row is not None and row[0] in {"table", "view", "virtual", "shadow"}:
+            cursor = conn.execute(query)
-                return row[0]
+            columns = [description[0] for description in cursor.description]
+            for row in cursor.fetchall():
+                record = dict(zip(columns, row))
+                if record.get("name") != table:
+                    continue
+                if schema is not None and record.get("schema") != schema:
+                    continue
+                row_type = record.get("type")
+                if row_type in {"table", "view", "virtual", "shadow"}:
+                    return row_type
         except sqlite3.DatabaseError:
             pass
     return _sqlite_table_type_from_schema(conn, table, schema=schema)
 
 
+def check_structured_write_table(conn, table: str, *, allow_missing=False):
+    """Validate a row-write target on the connection that will perform the write."""
+    # SQLite resolves identifiers case-insensitively. The create API must not
+    # treat a differently cased existing name as a missing table.
+    row = conn.execute(
+        "select name from main.sqlite_master where name = ? collate nocase "
+        "and type in ('table', 'view')",
+        (table,),
+    ).fetchone()
+    if row is None and allow_missing:
+        return
+    if row is not None and sqlite_table_type(conn, row[0]) == "table":
+        return
+    # Virtual table modules can interpret row writes as administrative operations.
+    # Their shadow tables are internal storage, not independently writable data.
+    raise ValueError("Structured writes require an ordinary table")
+
+
 def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]:
     schema_table = _sqlite_schema_table(schema)
     try:
@@ -118,6 +161,63 @@ def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]
     return sorted(hidden_tables) + content_fts_tables
 
 
+def sqlite_derived_table_dependencies(
+    conn, *, schema: str | None = "main"
+) -> dict[str, str]:
+    """Return implementation table -> logical/content table dependencies.
+
+    ``PRAGMA table_list`` safely identifies virtual and shadow tables, but
+    does not report which virtual table owns a shadow table or which table is
+    named by an FTS ``content=`` option. Derive those relationships from
+    ``sqlite_master`` DDL and the documented shadow-table suffixes.
+
+    Database errors propagate: failed discovery must not be mistaken for an
+    empty dependency map and cached as permission to skip inheritance.
+    """
+    schema_table = _sqlite_schema_table(schema)
+    rows = conn.execute(
+        f"select name, sql from {schema_table} where type = 'table'"
+    ).fetchall()
+
+    table_names = {row[0] for row in rows}
+    # SQLite identifiers fold ASCII letters only.
+    identifier_case = str.maketrans(
+        "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
+    )
+    canonical_names = {name.translate(identifier_case): name for name in table_names}
+    dependencies = {}
+    for virtual_table, sql in rows:
+        module = _virtual_table_module(sql)
+        if module is None:
+            continue
+
+        # SQLite's documented shadow tables are implementation details of
+        # their logical virtual table.
+        for suffix in _VIRTUAL_TABLE_SHADOW_SUFFIXES.get(module, ()):
+            shadow_table = virtual_table + suffix
+            if shadow_table in table_names:
+                dependencies[shadow_table] = virtual_table
+
+        # An external-content FTS table can expose values fetched from its
+        # content table, so it must also depend on that table's permission.
+        if module in {"fts3", "fts4", "fts5"}:
+            content_table = _fts_external_content_table(sql)
+            if content_table:
+                dependencies[virtual_table] = content_table
+
+        if module in {"fts5vocab", "fts4aux"}:
+            source = _fts_vocabulary_source(sql, module, schema or "main")
+            source = (
+                canonical_names.get(source.translate(identifier_case))
+                if source
+                else None
+            )
+            # An unresolved source is itself derived, so the one-hop policy denies it.
+            dependencies[virtual_table] = source or virtual_table
+
+    return dependencies
+
+
 def _sqlite_table_type_from_schema(
     conn,
     table: str,
@@ -184,10 +284,151 @@ def _quote_identifier(value: str) -> str:
 def _virtual_table_module(sql: str | None) -> str | None:
     if not sql:
         return None
+    match = _VIRTUAL_TABLE_MODULE_RE.search(_strip_sql_comments(sql))
+    if match is None:
+        return None
+    return _unquote_sql_value(match.group(1)).lower()
+
+
+def _fts_external_content_table(sql: str | None) -> str | None:
+    """Extract the external ``content=`` table from an FTS declaration."""
+    if not sql:
+        return None
+    sql = _strip_sql_comments(sql)
     match = _VIRTUAL_TABLE_MODULE_RE.search(sql)
     if match is None:
         return None
-    return match.group(1).strip("\"'[]`").lower()
+    open_paren = sql.find("(", match.end())
+    if open_paren == -1:
+        return None
+    close_paren = sql.rfind(")")
+    if close_paren <= open_paren:
+        return None
+
+    for argument in _split_sql_arguments(sql[open_paren + 1 : close_paren]):
+        key, separator, value = argument.partition("=")
+        if not separator or key.strip().lower() != "content":
+            continue
+        return _unquote_sql_value(value.strip())
+    return None
+
+
+def _fts_vocabulary_source(sql: str, module: str, schema: str) -> str | None:
+    """Resolve a vocabulary source within the current SQLite schema.
+
+    Cross-schema sources cannot be represented by the dependency map and
+    are conservatively left unresolved.
+    """
+    sql = _strip_sql_comments(sql)
+    match = _VIRTUAL_TABLE_MODULE_RE.search(sql)
+    if match is None:
+        return None
+    start = sql.find("(", match.end())
+    end = sql.rfind(")")
+    if start < 0 or end <= start:
+        return None
+    arguments = [
+        _unquote_sql_value(arg.strip())
+        for arg in _split_sql_arguments(sql[start + 1 : end])
+    ]
+    expected = 2 if module == "fts5vocab" else 1
+    if len(arguments) == expected:
+        return arguments[0]
+    if len(arguments) == expected + 1 and arguments[0].lower() == schema.lower():
+        return arguments[1]
+    return None
+
+
+def _split_sql_arguments(arguments: str) -> list[str]:
+    """Split comma-separated SQLite arguments without splitting quoted text."""
+    parts = []
+    start = 0
+    quote = None
+    closing_quote = None
+    index = 0
+    while index < len(arguments):
+        char = arguments[index]
+        if quote is None:
+            if char in {"'", '"', "`", "["}:
+                quote = char
+                closing_quote = "]" if char == "[" else char
+            elif char == ",":
+                parts.append(arguments[start:index])
+                start = index + 1
+        elif char == closing_quote:
+            # Single/double/backtick quoting escapes the delimiter by
+            # doubling it. Square-bracket identifiers do not.
+            if (
+                quote != "["
+                and index + 1 < len(arguments)
+                and arguments[index + 1] == closing_quote
+            ):
+                index += 1
+            else:
+                quote = None
+                closing_quote = None
+        index += 1
+    parts.append(arguments[start:])
+    return parts
+
+
+def _strip_sql_comments(sql: str) -> str:
+    """Remove SQLite comments while preserving quoted strings/identifiers."""
+    output = []
+    quote = None
+    closing_quote = None
+    index = 0
+    while index < len(sql):
+        char = sql[index]
+        next_char = sql[index + 1] if index + 1 < len(sql) else ""
+        if quote is None:
+            if char in {"'", '"', "`", "["}:
+                quote = char
+                closing_quote = "]" if char == "[" else char
+                output.append(char)
+            elif char == "-" and next_char == "-":
+                index += 2
+                while index < len(sql) and sql[index] not in "\r\n":
+                    index += 1
+                output.append(" ")
+                continue
+            elif char == "/" and next_char == "*":
+                index += 2
+                while index + 1 < len(sql) and sql[index : index + 2] != "*/":
+                    index += 1
+                index = min(index + 2, len(sql))
+                output.append(" ")
+                continue
+            else:
+                output.append(char)
+        else:
+            output.append(char)
+            if char == closing_quote:
+                if (
+                    quote != "["
+                    and index + 1 < len(sql)
+                    and sql[index + 1] == closing_quote
+                ):
+                    output.append(sql[index + 1])
+                    index += 1
+                else:
+                    quote = None
+                    closing_quote = None
+        index += 1
+    return "".join(output)
+
+
+def _unquote_sql_value(value: str) -> str:
+    if len(value) < 2:
+        return value
+    pairs = {"'": "'", '"': '"', "`": "`", "[": "]"}
+    closing = pairs.get(value[0])
+    if closing is None or value[-1] != closing:
+        return value
+    unquoted = value[1:-1]
+    if value[0] != "[":
+        unquoted = unquoted.replace(closing * 2, closing)
+    return unquoted
 
 
 def _is_fts_content_virtual_table(sql: str | None) -> bool:

datasette/utils/testing.py

@@ -4,7 +4,7 @@ from urllib.parse import urlencode
 from asgiref.sync import async_to_sync
 
 # These wrapper classes pre-date the introduction of
-# datasette.client and httpx to Datasette. They could
+# datasette.client and httpx2 to Datasette. They could
 # be removed if the Datasette tests are modified to
 # call datasette.client directly.
 

datasette/version.py

@@ -1,2 +1,2 @@
-__version__ = "1.0a38"
+__version__ = "1.0a39"
 __version_info__ = tuple(__version__.split("."))

datasette/views/database.py

@@ -40,7 +40,11 @@ from datasette.write_sql import QueryWriteRejected
 
 from . import Context
 from .base import DatasetteError, View, stream_csv
-from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns
+from .query_helpers import (
+    _block_framing,
+    _ensure_stored_query_execution_permissions,
+    _table_columns,
+)
 from .table_create_alter import _create_table_ui_context
 from .table_extras import (
     QueryExtraContext,
@@ -857,7 +861,8 @@ class QueryView(View):
                 raise DatasetteError("?sql= is required", status=400)
 
             async def fetch_data_for_csv(request, _next=None):
-                results = await db.execute(sql, params, truncate=True)
+                # Reuse the trusted magic parameter values prepared above.
+                results = await db.execute(sql, params_for_query, truncate=True)
                 data = {"rows": results.rows, "columns": results.columns}
                 return data, None, None
 
@@ -1140,6 +1145,8 @@ class QueryView(View):
             assert False, f"Invalid format: {format_}"
         if datasette.cors:
             add_cors_headers(r.headers)
+        if stored_query_write and format_ == "html":
+            _block_framing(r)
         return r
 
 

datasette/views/execute_write.py

@@ -1,6 +1,7 @@
 import re
 from urllib.parse import urlencode
 
+from datasette.database import QueryInterrupted
 from datasette.resources import DatabaseResource
 from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3
 from datasette.utils.asgi import Response
@@ -384,7 +385,7 @@ class ExecuteWriteView(BaseView):
         try:
             execute_write_kwargs = {"request": request}
             cursor = await db.execute_write(sql, params, **execute_write_kwargs)
-        except sqlite3.DatabaseError as ex:
+        except (QueryInterrupted, sqlite3.DatabaseError) as ex:
             message = str(ex)
             if wants_json:
                 return _block_framing(Response.error([message], 400))

datasette/views/row.py

@@ -28,9 +28,11 @@ from datasette.utils import (
     path_with_format,
     path_with_removed_args,
     sqlite3,
+    tilde_decode,
     to_css_class,
 )
 from datasette.utils.asgi import Forbidden, NotFound, PayloadTooLarge, Response
+from datasette.utils.sqlite import check_structured_write_table
 
 from . import Context, from_extra
 from .base import BaseView, DatasetteError, stream_csv
@@ -137,6 +139,12 @@ class RowContext(Context):
     )
 
 
+async def _database_and_table_resource_from_request(datasette, request):
+    db = await datasette.resolve_database(request)
+    table = tilde_decode(request.url_vars["table"])
+    return db, table, TableResource(database=db.name, table=table)
+
+
 class RowView(BaseView):
     name = "row"
 
@@ -263,7 +271,7 @@ class RowView(BaseView):
         if ttl is None or not ttl.isdigit():
             ttl = self.ds.setting("default_cache_ttl")
 
-        return self.set_response_headers(response, ttl)
+        return self.set_response_headers(response, ttl, request)
 
     async def html(self, request, data, extra_template_data, templates):
         extras = {}
@@ -376,36 +384,50 @@ class RowView(BaseView):
             },
         )
 
-    def set_response_headers(self, response, ttl):
+    def set_response_headers(self, response, ttl, request=None):
+        private = getattr(request, "_datasette_private_response", False)
         # Set far-future cache expiry
         if self.ds.cache_headers and response.status == 200:
-            ttl = int(ttl)
+            if private:
-            if ttl == 0:
+                # This response is only visible to the current actor (denied
-                ttl_header = "no-cache"
+                # to anonymous requests), so it must never be stored by a
+                # shared cache/CDN - and ?_ttl= must not override that.
+                response.headers["Cache-Control"] = "private, no-store"
+                response.headers["Vary"] = "Cookie"
             else:
-                ttl_header = f"max-age={ttl}"
+                ttl = int(ttl)
-            response.headers["Cache-Control"] = ttl_header
+                if ttl == 0:
+                    ttl_header = "no-cache"
+                else:
+                    ttl_header = f"max-age={ttl}"
+                response.headers["Cache-Control"] = ttl_header
         response.headers["Referrer-Policy"] = "no-referrer"
         if self.ds.cors:
             add_cors_headers(response.headers)
         return response
 
     async def data(self, request, default_labels=False):
-        resolved = await self.ds.resolve_row(request)
+        db, table, resource = await _database_and_table_resource_from_request(
-        db = resolved.db
+            self.ds, request
+        )
         database = db.name
-        table = resolved.table
-        pk_values = resolved.pk_values
 
-        # Ensure user has permission to view this row
+        # Check the URL resource before resolving the row, so a denied request
+        # cannot distinguish an existing primary key from a missing one.
         visible, private = await self.ds.check_visibility(
             request.actor,
             action="view-table",
-            resource=TableResource(database=database, table=table),
+            resource=resource,
         )
         if not visible:
             raise Forbidden("You do not have permission to view this table")
+        # Record whether this response is private (visible to this actor
+        # only) so set_response_headers() can set appropriate Cache-Control
+        # headers, regardless of which output format ends up being rendered.
+        request._datasette_private_response = private
 
+        resolved = await self.ds.resolve_row(request)
+        pk_values = resolved.pk_values
         results = await resolved.db.execute(
             resolved.sql, resolved.params, truncate=True
         )
@@ -482,8 +504,8 @@ class RowView(BaseView):
             for row in display_rows:
                 for cell in row:
                     if cell["column"] in pk_set:
-                        cell["value"] = markupsafe.Markup(
+                        cell["value"] = markupsafe.Markup("<strong>{}</strong>").format(
-                            "<strong>{}</strong>".format(cell["value"])
+                            cell["value"]
                         )
 
             label_column = await db.label_column_for_table(table) if is_table else None
@@ -556,7 +578,7 @@ class RowView(BaseView):
                 "private": private,
                 "columns": reordered_columns,
                 "foreign_key_tables": await self.foreign_key_tables(
-                    database, table, pk_values
+                    database, table, pk_values, actor=request.actor
                 ),
                 "database_color": db.color,
                 "display_columns": display_columns,
@@ -633,12 +655,23 @@ class RowView(BaseView):
             ),
         )
 
-    async def foreign_key_tables(self, database, table, pk_values):
+    async def foreign_key_tables(self, database, table, pk_values, *, actor):
         if len(pk_values) != 1:
             return []
         db = self.ds.databases[database]
         all_foreign_keys = await db.get_all_foreign_keys()
-        foreign_keys = all_foreign_keys[table]["incoming"]
+        foreign_keys = []
+        table_permissions = {}
+        for fk in all_foreign_keys[table]["incoming"]:
+            other_table = fk["other_table"]
+            if other_table not in table_permissions:
+                table_permissions[other_table] = await self.ds.allowed(
+                    action="view-table",
+                    resource=TableResource(database=database, table=other_table),
+                    actor=actor,
+                )
+            if table_permissions[other_table]:
+                foreign_keys.append(fk)
         if len(foreign_keys) == 0:
             return []
 
@@ -695,9 +728,24 @@ def _truncated_row_flash_label(label):
     return label[: ROW_FLASH_LABEL_MAX_LENGTH - 1] + "\u2026"
 
 
-async def _row_flash_message(db, action, resolved, row=None):
+async def _row_flash_message(
+    datasette, request, action, resolved, row=None, *, refresh_row=False
+):
     pk_label = ", ".join(resolved.pk_values)
-    label_column = await db.label_column_for_table(resolved.table)
+    # Mutation permission does not grant access to stored row labels.
+    if not await datasette.allowed(
+        action="view-table",
+        resource=TableResource(database=resolved.db.name, table=resolved.table),
+        actor=request.actor,
+    ):
+        return f"{action} row {pk_label}"
+
+    if refresh_row and row is None:
+        results = await resolved.db.execute(
+            resolved.sql, resolved.params, truncate=True
+        )
+        row = results.first()
+    label_column = await resolved.db.label_column_for_table(resolved.table)
     label = row_label_from_label_column(row or resolved.row, label_column)
     if label:
         label = _truncated_row_flash_label(label)
@@ -710,22 +758,28 @@ async def _resolve_row_and_check_permission(datasette, request, permission):
     from datasette.app import DatabaseNotFound, RowNotFound, TableNotFound
 
     try:
-        resolved = await datasette.resolve_row(request)
+        _, _, resource = await _database_and_table_resource_from_request(
+            datasette, request
+        )
     except DatabaseNotFound as e:
         return False, Response.error([f"Database not found: {e.database_name}"], 404)
+
+    # Check the URL resource before resolving the row, so a denied request
+    # cannot distinguish an existing primary key from a missing one.
+    if not await datasette.allowed(
+        action=permission,
+        resource=resource,
+        actor=request.actor,
+    ):
+        return False, Response.error(["Permission denied"], 403)
+
+    try:
+        resolved = await datasette.resolve_row(request)
     except TableNotFound as e:
         return False, Response.error([f"Table not found: {e.table}"], 404)
     except RowNotFound as e:
         return False, Response.error([f"Record not found: {e.pk_values}"], 404)
 
-    # Ensure user has permission to delete this row
-    if not await datasette.allowed(
-        action=permission,
-        resource=TableResource(database=resolved.db.name, table=resolved.table),
-        actor=request.actor,
-    ):
-        return False, Response.error(["Permission denied"], 403)
-
     return True, resolved
 
 
@@ -744,6 +798,7 @@ class RowDeleteView(BaseView):
 
         # Delete table
         def delete_row(conn):
+            check_structured_write_table(conn, resolved.table)
             sqlite_utils.Database(conn)[resolved.table].delete(resolved.pk_values)
 
         try:
@@ -765,7 +820,7 @@ class RowDeleteView(BaseView):
             table_url = self.ds.urls.table(resolved.db.name, resolved.table)
             self.ds.add_message(
                 request,
-                await _row_flash_message(resolved.db, "Deleted", resolved),
+                await _row_flash_message(self.ds, request, "Deleted", resolved),
                 self.ds.INFO,
             )
             return Response.json({"ok": True, "redirect": str(table_url)}, status=200)
@@ -826,6 +881,7 @@ class RowUpdateView(BaseView):
             return Response.error(["Permission denied for alter-table"], 403)
 
         def update_row(conn):
+            check_structured_write_table(conn, resolved.table)
             sqlite_utils.Database(conn)[resolved.table].update(
                 resolved.pk_values, update, alter=alter
             )
@@ -838,7 +894,14 @@ class RowUpdateView(BaseView):
 
         result = {"ok": True}
         returned_row = None
-        if data.get("return"):
+        # Only read back and disclose the stored row if the actor is also
+        # allowed to view this table - update-row alone must not be usable
+        # to read data the actor cannot otherwise see.
+        if data.get("return") and await self.ds.allowed(
+            action="view-table",
+            resource=TableResource(database=resolved.db.name, table=resolved.table),
+            actor=request.actor,
+        ):
             results = await resolved.db.execute(
                 resolved.sql, resolved.params, truncate=True
             )
@@ -855,16 +918,15 @@ class RowUpdateView(BaseView):
         )
 
         if request.args.get("_message"):
-            message_row = returned_row
-            if message_row is None:
-                results = await resolved.db.execute(
-                    resolved.sql, resolved.params, truncate=True
-                )
-                message_row = results.first()
             self.ds.add_message(
                 request,
                 await _row_flash_message(
-                    resolved.db, "Updated", resolved, row=message_row
+                    self.ds,
+                    request,
+                    "Updated",
+                    resolved,
+                    row=returned_row,
+                    refresh_row=True,
                 ),
                 self.ds.INFO,
             )

datasette/views/special.py

@@ -311,6 +311,7 @@ class AllowedResourcesView(BaseView):
     has_json_alternate = False
 
     async def get(self, request):
+        await self.ds.ensure_permission(action="view-instance", actor=request.actor)
         await self.ds.refresh_schemas()
 
         # Check if user has permissions-debug (to show sensitive fields)
@@ -796,6 +797,8 @@ class CreateTokenView(BaseView):
             raise Forbidden(
                 "Token authentication cannot be used to create additional tokens"
             )
+        if "_r" in request.actor:
+            raise Forbidden("Restricted actors cannot create API tokens")
 
     async def shared(self, request):
         self.check_permission(request)
@@ -873,6 +876,11 @@ class CreateTokenView(BaseView):
                 else:
                     errors.append("Invalid expire duration unit")
 
+        if errors:
+            context = await self.shared(request)
+            context["errors"] = errors
+            return await self.render(["create_token.html"], request, context)
+
         # Are there any restrictions?
         from datasette.tokens import TokenRestrictions
 
@@ -1261,14 +1269,21 @@ class SchemaBaseView(BaseView):
 
     has_json_alternate = False
 
-    async def get_database_schema(self, database_name):
+    async def get_database_schema(self, database_name, actor):
         """Get schema SQL for a database."""
         db = self.ds.databases[database_name]
-        result = await db.execute(
+        allowed_tables_page = await self.ds.allowed_resources(
-            "select group_concat(sql, ';' || CHAR(10)) as schema from sqlite_master where sql is not null"
+            "view-table", actor, parent=database_name
+        )
+        allowed_table_names = {
+            resource.child async for resource in allowed_tables_page.all()
+        }
+        result = await db.execute(
+            "select tbl_name, sql from sqlite_master where sql is not null"
+        )
+        return ";\n".join(
+            row["sql"] for row in result.rows if row["tbl_name"] in allowed_table_names
         )
-        row = result.first()
-        return row["schema"] if row and row["schema"] else ""
 
     def format_json_response(self, data):
         """Format data as JSON response with CORS headers if needed."""
@@ -1330,7 +1345,7 @@ class InstanceSchemaView(SchemaBaseView):
         # Get schema for each database
         schemas = []
         for database_name in allowed_databases:
-            schema = await self.get_database_schema(database_name)
+            schema = await self.get_database_schema(database_name, request.actor)
             schemas.append({"database": database_name, "schema": schema})
 
         if format_ == "json":
@@ -1371,7 +1386,7 @@ class DatabaseSchemaView(SchemaBaseView):
         if database_name not in self.ds.databases:
             return self.format_error_response("Database not found", format_)
 
-        schema = await self.get_database_schema(database_name)
+        schema = await self.get_database_schema(database_name, request.actor)
 
         if format_ == "json":
             return self.format_json_response(
@@ -1410,7 +1425,8 @@ class TableSchemaView(SchemaBaseView):
         # Get schema for the table
         db = self.ds.databases[database_name]
         result = await db.execute(
-            "select sql from sqlite_master where name = ? and sql is not null",
+            "select sql from sqlite_master where name = ? "
+            "and type in ('table', 'view') and sql is not null",
             [table_name],
         )
         row = result.first()

datasette/views/stored_queries.py

@@ -279,7 +279,7 @@ class QueryCreateView(BaseView):
             ),
         )
         response.status = status
-        return response
+        return _block_framing(response)
 
     async def get(self, request):
         db = await self.ds.resolve_database(request)
@@ -527,7 +527,7 @@ class QueryEditView(BaseView):
             ),
         )
         response.status = status
-        return response
+        return _block_framing(response)
 
     async def get(self, request):
         db, query_name, existing = await self._load(request)
@@ -639,15 +639,17 @@ class QueryDeleteView(BaseView):
             return Response.error(
                 ["Trusted queries cannot be deleted using the API"], 403
             )
-        return await self.render(
+        return _block_framing(
-            ["query_delete.html"],
+            await self.render(
-            request,
+                ["query_delete.html"],
-            {
+                request,
-                "database": db.name,
+                {
-                "database_color": db.color,
+                    "database": db.name,
-                "query": stored_query_to_dict(existing),
+                    "database_color": db.color,
-                "query_url": self.ds.urls.table(db.name, query_name),
+                    "query": stored_query_to_dict(existing),
-            },
+                    "query_url": self.ds.urls.table(db.name, query_name),
+                },
+            )
         )
 
     async def post(self, request):

datasette/views/table.py

@@ -57,6 +57,7 @@ from datasette.utils.asgi import (
     Request,
     Response,
 )
+from datasette.utils.sqlite import check_structured_write_table
 
 from . import Context, from_extra
 from .base import BaseView, DatasetteError, stream_csv
@@ -1126,6 +1127,7 @@ class TableInsertView(BaseView):
             row_pk_values_for_later = [tuple(row[pk] for pk in pks) for row in rows]
 
         def insert_or_upsert_rows(conn):
+            check_structured_write_table(conn, table_name)
             table = sqlite_utils.Database(conn)[table_name]
             kwargs = {}
             if upsert:
@@ -1157,17 +1159,32 @@ class TableInsertView(BaseView):
             # TODO: narrow to expected write errors so Datasette bugs surface as 500s
             return Response.error([str(e)])
         result = {"ok": True}
+        # Only read back and disclose stored rows if the actor is also
+        # allowed to view this table - insert-row/update-row alone must
+        # not be usable to read data the actor cannot otherwise see.
+        if should_return and not await self.ds.allowed(
+            action="view-table",
+            resource=TableResource(database=database_name, table=table_name),
+            actor=request.actor,
+        ):
+            should_return = False
         if should_return:
             if upsert:
                 # Fetch based on initial input IDs
                 where_clause = " OR ".join(
-                    ["({})".format(" AND ".join(f"{pk} = ?" for pk in pks))]
+                    [
+                        "({})".format(
+                            " AND ".join(f"{escape_sqlite(pk)} = ?" for pk in pks)
+                        )
+                    ]
                     * len(row_pk_values_for_later)
                 )
                 args = list(itertools.chain.from_iterable(row_pk_values_for_later))
                 fetched_rows = await db.execute(
-                    "select {}* from [{}] where {}".format(
+                    "select {}* from {} where {}".format(
-                        "rowid, " if pks == ["rowid"] else "", table_name, where_clause
+                        "rowid, " if pks == ["rowid"] else "",
+                        escape_sqlite(table_name),
+                        where_clause,
                     ),
                     args,
                 )
@@ -1382,7 +1399,9 @@ class TableDropView(BaseView):
                     "database": database_name,
                     "table": table_name,
                     "row_count": (
-                        await db.execute(f"select count(*) from [{table_name}]")
+                        await db.execute(
+                            f"select count(*) from {escape_sqlite(table_name)}"
+                        )
                     ).single_value(),
                     "message": 'Pass "confirm": true to confirm',
                 },
@@ -1391,7 +1410,9 @@ class TableDropView(BaseView):
 
         # Drop table
         def drop_table(conn):
-            sqlite_utils.Database(conn)[table_name].drop()
+            table = sqlite_utils.Database(conn)[table_name]
+            table.disable_fts()
+            table.drop()
 
         await db.execute_write_fn(drop_table, request=request)
         await self.ds.track_event(
@@ -1407,6 +1428,42 @@ class TableDropView(BaseView):
         return Response.json({"ok": True}, status=200)
 
 
+class TableCountView(BaseView):
+    name = "table-count"
+
+    async def post(self, request):
+        try:
+            return await self.count(request)
+        except (NotFound, Forbidden, BadRequest, DatasetteError) as ex:
+            return Response.error(str(ex), status=ex.status)
+
+    async def count(self, request):
+        resolved = await self.ds.resolve_table(request)
+        visible, _private = await self.ds.check_visibility(
+            request.actor,
+            action="view-table",
+            resource=TableResource(database=resolved.db.name, table=resolved.table),
+        )
+        if not visible:
+            raise Forbidden("You do not have permission to view this table")
+        _, where_clauses, params, _, _ = await _table_filters(
+            self.ds, request, resolved.db.name, resolved.table
+        )
+        sql = f"select count(*) from {escape_sqlite(resolved.table)}"
+        if where_clauses:
+            sql += " where " + " and ".join(where_clauses)
+        try:
+            results = await resolved.db.execute(sql, params)
+        except QueryInterrupted:
+            return Response.error("Count query timed out", status=400)
+        except (sqlite3.OperationalError, InvalidSql) as ex:
+            return Response.error(str(ex), status=400)
+        return Response.json(
+            {"ok": True, "count": results.single_value()},
+            headers={"Cache-Control": "no-store"},
+        )
+
+
 class TableFragmentView(BaseView):
     name = "table-fragment"
 
@@ -1693,13 +1750,22 @@ async def table_view(datasette, request):
     if ttl is None or not ttl.isdigit():
         ttl = datasette.setting("default_cache_ttl")
 
+    private = getattr(request, "_datasette_private_response", False)
+
     if datasette.cache_headers and response.status == 200:
-        ttl = int(ttl)
+        if private:
-        if ttl == 0:
+            # This response is only visible to the current actor (denied to
-            ttl_header = "no-cache"
+            # anonymous requests), so it must never be stored by a shared
+            # cache/CDN - and ?_ttl= must not be able to override that.
+            response.headers["Cache-Control"] = "private, no-store"
+            response.headers["Vary"] = "Cookie"
         else:
-            ttl_header = f"max-age={ttl}"
+            ttl = int(ttl)
-        response.headers["Cache-Control"] = ttl_header
+            if ttl == 0:
+                ttl_header = "no-cache"
+            else:
+                ttl_header = f"max-age={ttl}"
+            response.headers["Cache-Control"] = ttl_header
 
     # Referrer policy
     response.headers["Referrer-Policy"] = "no-referrer"
@@ -1923,6 +1989,47 @@ async def table_view_traced(datasette, request):
     return r
 
 
+async def _table_filters(datasette, request, database_name, table_name):
+    # Arguments that start with _ and don't contain a __ are
+    # special - things like ?_search= - and should not be
+    # treated as filters.
+    filter_args = []
+    for key in request.args:
+        if not (key.startswith("_") and "__" not in key):
+            for v in request.args.getlist(key):
+                filter_args.append((key, v))
+
+    # Build where clauses from query string arguments
+    filters = Filters(sorted(filter_args))
+    where_clauses, params = filters.build_where_clauses(table_name)
+
+    # Execute filters_from_request plugin hooks - including the default
+    # ones that live in datasette/filters.py
+    extra_context_from_filters = {}
+    extra_human_descriptions = []
+
+    for hook in pm.hook.filters_from_request(
+        request=request,
+        table=table_name,
+        database=database_name,
+        datasette=datasette,
+    ):
+        filter_arguments = await await_me_maybe(hook)
+        if filter_arguments:
+            where_clauses.extend(filter_arguments.where_clauses)
+            params.update(filter_arguments.params)
+            extra_human_descriptions.extend(filter_arguments.human_descriptions)
+            extra_context_from_filters.update(filter_arguments.extra_context)
+
+    return (
+        filters,
+        where_clauses,
+        params,
+        extra_human_descriptions,
+        extra_context_from_filters,
+    )
+
+
 async def table_view_data(
     datasette,
     request,
@@ -1947,6 +2054,10 @@ async def table_view_data(
     )
     if not visible:
         raise Forbidden("You do not have permission to view this table")
+    # Record whether this response is private (visible to this actor only)
+    # so the outer table_view() can set appropriate Cache-Control headers,
+    # regardless of which output format ends up being rendered.
+    request._datasette_private_response = private
 
     # Redirect based on request.args, if necessary
     redirect_response = await _redirect_if_needed(datasette, request, resolved)
@@ -1997,36 +2108,13 @@ async def table_view_data(
 
     table_metadata = await datasette.table_config(database_name, table_name)
 
-    # Arguments that start with _ and don't contain a __ are
+    (
-    # special - things like ?_search= - and should not be
+        filters,
-    # treated as filters.
+        where_clauses,
-    filter_args = []
+        params,
-    for key in request.args:
+        extra_human_descriptions,
-        if not (key.startswith("_") and "__" not in key):
+        extra_context_from_filters,
-            for v in request.args.getlist(key):
+    ) = await _table_filters(datasette, request, database_name, table_name)
-                filter_args.append((key, v))
-
-    # Build where clauses from query string arguments
-    filters = Filters(sorted(filter_args))
-    where_clauses, params = filters.build_where_clauses(table_name)
-
-    # Execute filters_from_request plugin hooks - including the default
-    # ones that live in datasette/filters.py
-    extra_context_from_filters = {}
-    extra_human_descriptions = []
-
-    for hook in pm.hook.filters_from_request(
-        request=request,
-        table=table_name,
-        database=database_name,
-        datasette=datasette,
-    ):
-        filter_arguments = await await_me_maybe(hook)
-        if filter_arguments:
-            where_clauses.extend(filter_arguments.where_clauses)
-            params.update(filter_arguments.params)
-            extra_human_descriptions.extend(filter_arguments.human_descriptions)
-            extra_context_from_filters.update(filter_arguments.extra_context)
 
     # Deal with custom sort orders
     sortable_columns = await _sortable_columns_for_table(
@@ -2430,9 +2518,12 @@ async def _next_value_and_url(
             except IndexError:
                 # sort/sort_desc column missing from SELECT - look up value by PK instead
                 prefix_where_clause = " and ".join(
-                    f"[{pk}] = :pk{i}" for i, pk in enumerate(pks)
+                    f"{escape_sqlite(pk)} = :pk{i}" for i, pk in enumerate(pks)
+                )
+                prefix_lookup_sql = (
+                    f"select {escape_sqlite(sort or sort_desc)} "
+                    f"from {escape_sqlite(table_name)} where {prefix_where_clause}"
                 )
-                prefix_lookup_sql = f"select [{sort or sort_desc}] from [{table_name}] where {prefix_where_clause}"
                 prefix = (
                     await db.execute(
                         prefix_lookup_sql,

datasette/views/table_create_alter.py

@@ -27,7 +27,15 @@ from datasette.utils import (
     table_column_details,
 )
 from datasette.utils.asgi import NotFound, PayloadTooLarge, Response
-from datasette.utils.sqlite import sqlite_hidden_table_names
+from datasette.utils.permissions import (
+    SKIP_PERMISSION_CHECKS,
+    gather_permission_sql_from_hooks,
+    resolve_permissions_with_candidates,
+)
+from datasette.utils.sqlite import (
+    check_structured_write_table,
+    sqlite_hidden_table_names,
+)
 
 from .base import BaseView
 
@@ -122,6 +130,30 @@ def _public_foreign_key_target(target):
     }
 
 
+async def _filter_visible_foreign_key_targets(datasette, actor, database_name, targets):
+    if not targets:
+        return []
+
+    permission_sqls = await gather_permission_sql_from_hooks(
+        datasette=datasette,
+        actor=actor,
+        action="view-table",
+    )
+    if permission_sqls is SKIP_PERMISSION_CHECKS:
+        return targets
+
+    candidate_tables = list(dict.fromkeys(target["fk_table"] for target in targets))
+    permission_rows = await resolve_permissions_with_candidates(
+        datasette.get_internal_database(),
+        actor,
+        permission_sqls,
+        [(database_name, table_name) for table_name in candidate_tables],
+        "view-table",
+    )
+    visible_tables = {row["child"] for row in permission_rows if bool(row["allow"])}
+    return [target for target in targets if target["fk_table"] in visible_tables]
+
+
 def _singular(name):
     if name.endswith("ies") and len(name) > 3:
         return name[:-3] + "y"
@@ -821,16 +853,18 @@ class TableCreateView(BaseView):
         ignore = create_request.ignore
         replace = create_request.replace
 
+        table_name = create_request.table
+        table_exists = await db.table_exists(table_name)
+        table_resource = TableResource(database=database_name, table=table_name)
+
         # Replacing rows requires update-row permission
         if replace and not await self.ds.allowed(
             action="update-row",
-            resource=DatabaseResource(database=database_name),
+            resource=table_resource,
             actor=request.actor,
         ):
             return Response.error(["Permission denied: need update-row"], 403)
 
-        table_name = create_request.table
-        table_exists = await db.table_exists(table_name)
         columns = create_request.columns
         rows = create_request.rows_list
 
@@ -838,7 +872,7 @@ class TableCreateView(BaseView):
             # Must have insert-row permission
             if not await self.ds.allowed(
                 action="insert-row",
-                resource=DatabaseResource(database=database_name),
+                resource=table_resource,
                 actor=request.actor,
             ):
                 return Response.error(["Permission denied: need insert-row"], 403)
@@ -857,7 +891,7 @@ class TableCreateView(BaseView):
                 if create_request.alter:
                     if not await self.ds.allowed(
                         action="alter-table",
-                        resource=DatabaseResource(database=database_name),
+                        resource=table_resource,
                         actor=request.actor,
                     ):
                         return Response.error(
@@ -893,6 +927,7 @@ class TableCreateView(BaseView):
             )
 
         def create_table(conn):
+            check_structured_write_table(conn, table_name, allow_missing=True)
             db_for_write = sqlite_utils.Database(conn)
             table = db_for_write[table_name]
             if rows:
@@ -1012,6 +1047,9 @@ class DatabaseForeignKeyTargetsView(BaseView):
             for target in (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts()
             if target["fk_table"] not in hidden_tables
         ]
+        targets = await _filter_visible_foreign_key_targets(
+            self.ds, request.actor, database_name, targets
+        )
         return Response.json(
             {
                 "ok": True,
@@ -1050,6 +1088,15 @@ class TableForeignKeySuggestionsView(BaseView):
         source_columns, targets, current_by_column = await db.execute_fn(
             lambda conn: _foreign_key_suggestion_metadata(conn, table_name)
         )
+        targets = await _filter_visible_foreign_key_targets(
+            self.ds, request.actor, database_name, targets
+        )
+        visible_target_tables = {target["fk_table"] for target in targets}
+        current_by_column = {
+            column: current
+            for column, current in current_by_column.items()
+            if current["fk_table"] in visible_target_tables
+        }
 
         columns = []
         options_by_column = {}

datasette/views/table_extras.py

@@ -1206,7 +1206,10 @@ class ForeignKeyTablesExtra(Extra):
 
     async def resolve(self, context):
         return await context.foreign_key_tables(
-            context.database_name, context.table_name, context.pk_values
+            context.database_name,
+            context.table_name,
+            context.pk_values,
+            actor=context.request.actor,
         )
 
 

datasette/write_sql.py

@@ -83,6 +83,22 @@ def decision_for_write_sql_operation(
             )
     if operation.operation == "function":
         return IgnoreWriteSqlOperation("SQL function")
+    if (
+        operation.operation == "read"
+        and operation.target_type == "table"
+        and operation.table is not None
+        and operation.table_kind is None
+        and operation.table.lower().startswith("pragma_")
+    ):
+        # Eponymous table-valued PRAGMA functions (e.g. pragma_table_info("secret"))
+        # report a read of the synthetic "pragma_table_info" table, not of the
+        # table passed as an argument. That means a view-table denial on the real
+        # table is never consulted, so these could otherwise be used to read
+        # schema metadata (column names, table lists, ...) for tables the actor
+        # is not allowed to view. Reject them outright in untrusted write SQL,
+        # including inside CREATE VIEW bodies (whose reads are discovered here
+        # via the rolled-back dependency-read analysis above).
+        return UnsupportedWriteSqlOperation(unsupported_message)
     if (
         operation.operation == "read"
         and operation.target_type == "table"

docs/authentication.rst

@@ -158,6 +158,15 @@ Datasette resolves matching rules from most specific to least specific:
 
 This means a resource-level allow can provide an exception to a parent-level deny. It also means that two plugins which disagree at the same level resolve to deny.
 
+For table and view permissions, resource names use SQLite's case-insensitive
+identifier matching: ``Secret``, ``secret`` and ``SECRET`` identify the same
+table. This applies to configuration rules, plugin rules and token restrictions.
+Only ASCII letters are case-insensitive; non-ASCII characters remain distinct.
+Conflicting rules for different spellings of the same name follow the usual
+deny-wins rule at the same scope. Names retain their original spelling in
+resource listings and permission explanations. Database names, stored query
+names and other resource types remain case-sensitive.
+
 .. list-table:: Permission rule examples
    :header-rows: 1
 
@@ -182,6 +191,18 @@ This means a resource-level allow can provide an exception to a parent-level den
 
 The built-in public defaults are global allow rules for actions such as ``view-instance``, ``view-database`` and ``view-table``. They follow the same precedence rules as configuration and plugin rules. The ``--default-deny`` option prevents Datasette from contributing those default allow rules.
 
+The built-in ``datasette.default_permissions.sqlite_statistics`` plugin denies
+``view-table`` for ``sqlite_stat1``, ``sqlite_stat2``, ``sqlite_stat3`` and
+``sqlite_stat4``. These table-level denials also apply to root users and take
+precedence over configuration or plugin allow rules at the same scope.
+This controls table access and listings, without changing ``execute-sql`` or
+SQLite's internal use of statistics.
+
+A plugin can replace this policy by unregistering
+``datasette.default_permissions.sqlite_statistics`` through ``datasette.pm``
+and registering its own permission hook. Plugin registration is process-wide:
+replacing this policy affects every Datasette instance in that process.
+
 Datasette performs checks using :ref:`datasette_allowed`, which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``.
 
 ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified.
@@ -771,6 +792,8 @@ Datasette defaults to allowing any site visitor to execute their own custom SQL
 
 Access to this ability is controlled by the :ref:`actions_execute_sql` permission.
 
+This permission does not apply to structured table-browsing operations where Datasette constructs the SQL, such as sorting, column filters and :ref:`facets`. Faceting is controlled separately by the :ref:`setting_allow_facet` setting.
+
 The easiest way to disable arbitrary SQL queries is using the :ref:`default_allow_sql setting <setting_default_allow_sql>` when you first start Datasette running.
 
 You can alternatively use an ``"allow_sql"`` block to control who is allowed to execute arbitrary SQL queries.
@@ -1359,6 +1382,12 @@ view-table
 
 Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.io/fixtures/complex_foreign_keys
 
+Derived implementation tables require access to their immediate source: FTS and RTree shadow tables require access to their virtual table, external-content FTS tables require access to their content table, and FTS vocabulary tables (``fts5vocab`` and ``fts4aux``) require access to their FTS table. The derived table's own permission rules also apply.
+
+Access is always denied if the source table is itself derived, or if a vocabulary table's source cannot be identified.
+
+The same rules apply to individual permission checks and table listings, including whether they are private. If a database error prevents dependency discovery, the check or listing fails with an error instead of ignoring the dependencies. Failed discovery results are not cached, so later checks can retry.
+
 ``resource`` - ``datasette.resources.TableResource(database, table)``
     ``database`` is the name of the database (string)
 
@@ -1521,6 +1550,8 @@ execute-sql
 
 Actor is allowed to run arbitrary read-only SQL queries against a specific database using the :ref:`custom SQL query page <pages_custom_sql_queries>`, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100
 
+This action also controls raw SQL supplied using ``?_where=``. It does not control structured table-browsing features such as :ref:`facets`, which use SQL generated by Datasette and are controlled by :ref:`setting_allow_facet`.
+
 ``resource`` - ``datasette.resources.DatabaseResource(database)``
     ``database`` is the name of the database (string)
 

docs/changelog.rst

@@ -4,6 +4,67 @@
 Changelog
 =========
 
+.. _unreleased:
+
+Unreleased
+----------
+
+- Fixed incorrect counts when clicking **count all** on filtered table pages. The button now uses a new :ref:`POST count endpoint <TableCountView>`. (:issue:`2914`)
+- Datasette now uses `httpx2 <https://httpx2.pydantic.dev/>`__, the Pydantic-maintained continuation of `httpx <https://www.python-httpx.org/>`__, in place of ``httpx``. The public API is the same, but responses returned by :ref:`internals_datasette_client` are now ``httpx2.Response`` objects rather than ``httpx.Response``. Plugins that use ``isinstance()`` checks against ``httpx.Response`` should be updated to use ``httpx2``. **Plugins that use httpx without explicitly depending on it** will need to add an explicit dependency or switch to `httpx2`.
+
+Background tasks
+~~~~~~~~~~~~~~~~
+
+Datasette plugins can now use **background tasks** to run code independent of the Datasette request/response cycle.
+
+- New :ref:`datasette_add_background_task` API: plugins register supervised, long-lived background work - typically from a ``startup`` hook - and these will be launched after every ``startup`` hook has run. Tasks are cancelled (with a five-second grace period) on shutdown.
+- New ``/-/tasks`` JSON debug endpoint lists every supervised background task and its state, in the style of ``/-/threads``. See :ref:`JsonDataView_tasks`. It requires the ``permissions-debug`` permission.
+- New :ref:`plugin_hook_shutdown` plugin hook, called during graceful shutdown (Ctrl-C, ``SIGTERM``) before background tasks are cancelled and before database connections are closed. It is not called on a hard kill (``SIGKILL``).
+- Plugin ``asgi_wrapper`` middleware now always runs *after* startup has completed.
+- If your plugin uses ``asgi_wrapper`` to start background tasks on the first incoming request, you should migrate to ``datasette.add_background_task()`` instead. `datasette-cron <https://datasette.io/plugins/datasette-cron>`__ and `datasette-enrichments <https://datasette.io/plugins/datasette-enrichments>`__ are being migrated to this pattern.
+
+.. _v1_0_a39:
+
+1.0a39 (2026-09-10)
+-------------------
+
+This alpha release includes security fixes for permissions, SQL construction, HTML rendering, authentication and caching, plus improvements to application startup and write execution.
+
+See `0.65.4 <https://docs.datasette.io/en/stable/changelog.html#v0-65-4>`__ for fixes that have been backported to the stable 0.65.x branch.
+
+The Datasette blog `has more details on these releases <https://datasette.io/blog/2026/september-security-releases/>`__.
+
+Some of the security fixes include:
+
+- Table and view permission checks now take SQLite's case-insensitive names into account. See :ref:`authentication_permissions_explained`.
+- Viewing a full-text search index table now checks you have permission to view the table from which it draws its content.
+- Viewing SQLite statistics tables (``sqlite_stat1`` through ``sqlite_stat4``) is now denied by a default.
+- Table schema display now obeys the ``view-table`` permission.
+- Table filters using ``?_through=`` require permission to view the intermediate table.
+- Foreign-key target and suggestion APIs, incoming foreign-key relationships and their row counts now respect ``view-table`` permission.
+- Row endpoints check permissions before resolving primary keys, to avoid revealing the existence of an otherwise invisible primary key.
+- Improved permission checks for the create-table API. See :ref:`json_api_write`.
+- The write SQL interface now checks ``view-table`` permission for tables referenced by ``CREATE VIEW`` statements.
+- Fixed SQL identifier escaping for column names from untrusted database schemas.
+- Fixed HTML escaping for column names from untrusted database schemas.
+- URL columns now render links only for validated HTTP or HTTPS URLs.
+- Private and personalized dynamic responses now use ``Cache-Control: private, no-store``. Anonymous dynamic responses vary by ``Cookie`` and ``Authorization``.
+- Actor cookies now respect ``expire_after``.
+- Restricted actors can no longer create API tokens.
+- Stored-query create, edit and delete forms now block framing to prevent clickjacking.
+- Configuration secret redaction now matches key names case-insensitively.
+- SQLite extension loading is disabled after extensions supplied using ``--load-extension`` have been loaded.
+
+Other improvements and fixes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- :ref:`db.execute_write() <database_execute_write>` now has a default execution time limit of 2,000ms. Plugins can override this using ``time_limit_ms=`` or disable it using ``time_limit_ms=None``. This limit is independent of the ``sql_time_limit_ms`` setting for read queries.
+- Application startup now runs through ASGI lifespan events before requests are accepted, with a first-request fallback for hosts without lifespan support. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:pr:`2887`)
+- ``datasette serve`` now runs startup hooks and Uvicorn on the same event loop, preserving background tasks started by plugins. The minimum Uvicorn version is now 0.29. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:pr:`2886`)
+- Non-blocking writes using ``execute_write_fn(..., block=False)`` now return a distinct task UUID for every call and work correctly with ``num_sql_threads=0``. Thanks, `Zain Dana Harper <https://github.com/HarperZ9>`__. (:issue:`2860`, :issue:`2859`)
+- Dropping a table now disables its full-text search index first. (:issue:`2874`)
+- Fixed ``CREATE VIEW`` SQL analysis on Python 3.10.
+
 .. _v1_0_a38:
 
 1.0a38 (2026-08-06)
@@ -1092,7 +1153,7 @@ Features
 - New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`)
 - Spaces in the database names in URLs are now encoded as ``+`` rather than ``~20``. (:issue:`1701`)
 - ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`)
-- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image <https://hub.docker.com/datasetteproject/datasette>`__ has been upgraded to ``3.10.6-slim-bullseye``.  (:issue:`1768`)
+- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image <https://hub.docker.com/r/datasetteproject/datasette>`__ has been upgraded to ``3.10.6-slim-bullseye``.  (:issue:`1768`)
 - Canned writable queries against immutable databases now show a warning message. (:issue:`1728`)
 - ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (:pr:`1717`)
 - ``datasette publish cloudrun`` has new ``--min-instances`` and ``--max-instances`` options. (:issue:`1779`)
@@ -2121,7 +2182,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you
 - Removed obsolete ``?_group_count=col`` feature (:issue:`504`)
 - Improved user interface and documentation for ``datasette publish cloudrun`` (:issue:`608`)
 - Tables with indexes now show the ``CREATE INDEX`` statements on the table page (:issue:`618`)
-- Current version of `uvicorn <https://www.uvicorn.org/>`__ is now shown on ``/-/versions``
+- Current version of `uvicorn <https://uvicorn.dev/>`__ is now shown on ``/-/versions``
 - Python 3.8 is now supported! (:issue:`622`)
 - Python 3.5 is no longer supported.
 
@@ -2172,7 +2233,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you
 0.29.2 (2019-07-13)
 -------------------
 
-- Bumped `Uvicorn <https://www.uvicorn.org/>`__ to 0.8.4, fixing a bug where the query string was not included in the server logs. (:issue:`559`)
+- Bumped `Uvicorn <https://uvicorn.dev/>`__ to 0.8.4, fixing a bug where the query string was not included in the server logs. (:issue:`559`)
 - Fixed bug where the navigation breadcrumbs were not displayed correctly on the page for a custom query. (:issue:`558`)
 - Fixed bug where custom query names containing unicode characters caused errors.
 
@@ -2194,7 +2255,7 @@ ASGI, new plugin hooks, facet by date and much, much more...
 ASGI
 ~~~~
 
-`ASGI <https://asgi.readthedocs.io/>`__ is the Asynchronous Server Gateway Interface standard. I've been wanting to convert Datasette into an ASGI application for over a year - `Port Datasette to ASGI #272 <https://github.com/simonw/datasette/issues/272>`__ tracks thirteen months of intermittent development - but with Datasette 0.29 the change is finally released. This also means Datasette now runs on top of `Uvicorn <https://www.uvicorn.org/>`__ and no longer depends on `Sanic <https://github.com/huge-success/sanic>`__.
+`ASGI <https://asgi.readthedocs.io/>`__ is the Asynchronous Server Gateway Interface standard. I've been wanting to convert Datasette into an ASGI application for over a year - `Port Datasette to ASGI #272 <https://github.com/simonw/datasette/issues/272>`__ tracks thirteen months of intermittent development - but with Datasette 0.29 the change is finally released. This also means Datasette now runs on top of `Uvicorn <https://uvicorn.dev/>`__ and no longer depends on `Sanic <https://github.com/huge-success/sanic>`__.
 
 I wrote about the significance of this change in `Porting Datasette to ASGI, and Turtles all the way down <https://simonwillison.net/2019/Jun/23/datasette-asgi/>`__.
 

docs/facets.rst

@@ -14,6 +14,8 @@ Here's `an example <https://congress-legislators.datasettes.com/legislators/legi
 
 Facets can be specified in two ways: using query string parameters, or in ``metadata.json`` configuration for the table.
 
+Facet queries are generated by Datasette and summarize rows the actor already has permission to view. They do not require the :ref:`actions_execute_sql` permission. Use the :ref:`setting_allow_facet` setting to control whether users can request facets using query string parameters.
+
 Facets in query strings
 -----------------------
 

docs/full_text_search.rst

@@ -52,11 +52,11 @@ Configuring full-text search for a table or view
 
 If a table has a corresponding FTS table set up using the ``content=`` argument to ``CREATE VIRTUAL TABLE`` shown below, Datasette will detect it automatically and add a search interface to the table page for that table.
 
-You can also manually configure which table should be used for full-text search using query string parameters or table configuration in ``datasette.yaml`` (see :ref:`table_configuration_fts`). You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option.
+You can also manually configure which table should be used for full-text search using table configuration in ``datasette.yaml`` (see :ref:`table_configuration_fts`). You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option.
 
-Use ``?_fts_table=x`` to over-ride the FTS table for a specific page. If the primary key was something other than ``rowid`` you can use ``?_fts_pk=col`` to set that as well. This is particularly useful for views, for example:
+The legacy ``?_fts_table=x`` and ``?_fts_pk=col`` query string parameters are accepted only if they exactly match the configured or automatically detected FTS mapping. They cannot be used to select a different FTS table or primary key. This prevents a public table from being used to probe the contents of a private FTS table.
 
-https://latest.datasette.io/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk
+Searching also requires the current actor to have ``view-table`` permission for the FTS table itself, in addition to permission to view the table or view being searched.
 
 The ``fts_table`` metadata property can be used to specify an associated FTS table. If the primary key column in your table which was used to populate the FTS table is something other than ``rowid``, you can specify the column to use with the ``fts_pk`` property.
 

docs/internals.rst

@@ -1403,7 +1403,139 @@ Release all resources held by this ``Datasette`` instance. This calls :ref:`data
 
 If a call to ``Database.close()`` on one of the attached databases raises an exception, ``Datasette.close()`` will continue trying to close the remaining databases and will re-raise the first exception after every database has been processed.
 
-When Datasette is being served over ASGI the ``close()`` method is wired up to the lifespan shutdown event, so resources are released cleanly on ``SIGTERM`` / ``SIGINT``.
+When Datasette is being served over ASGI the ``close()`` method is wired up to the lifespan shutdown event, so resources are released cleanly on ``SIGTERM`` / ``SIGINT``. See :ref:`datasette_lifecycle` for where ``close()`` fits into the full startup-to-shutdown sequence.
+
+.. _datasette_lifecycle:
+
+Application lifecycle
+---------------------
+
+Datasette guarantees a fixed sequence of events between the moment a ``Datasette`` instance is constructed and the moment its resources are released:
+
+1. ``Datasette(...)`` — the constructor runs synchronously and does not run plugin hooks.
+2. **Startup** — ``await datasette.invoke_startup()`` runs once: it populates the internal database's catalog of table schemas (:ref:`internals_internal`), loads canned queries and column type configuration, then calls every registered :ref:`plugin_hook_startup` hook, in plugin registration order. When Datasette is being served, table-count precomputation for immutable databases runs immediately before this, as part of the same startup sequence.
+3. **Background-task launch** — once *every* ``startup`` hook has finished (not before), every task registered with :ref:`datasette_add_background_task` — by any plugin — is launched. A task registered by one plugin's ``startup`` hook can safely depend on state set up by another plugin's ``startup`` hook, because launch only happens after the whole round of hooks completes.
+4. **Serving** — the instance handles requests (or, for headless or CLI use, does whatever the embedding program does with it).
+5. **Shutdown** — triggered by the ASGI ``lifespan.shutdown`` event (Ctrl-C, ``SIGTERM``) or the end of a ``datasette serve`` process: every :ref:`plugin_hook_shutdown` hook runs first, while background tasks are still alive, so a plugin can tell its own task to wind down gracefully; every still-running background task is then cancelled and given a five-second grace period to actually stop; finally every database connection is released via :ref:`datasette_close`.
+
+.. admonition:: Startup hooks run on the event loop that serves requests
+
+   In every trigger path below, ``startup`` hooks run on the same ``asyncio`` event loop that goes on to accept connections. It is safe to create loop-bound primitives — ``asyncio.Lock``, ``asyncio.Queue``, ``asyncio.Event``, a raw ``asyncio.create_task()`` call — inside a ``startup`` hook, and to register long-lived background work with :ref:`datasette_add_background_task` there. This was not always true: older Datasette versions ran startup on a temporary event loop in the CLI that was closed before the server's own loop was created, which could silently kill anything scheduled on it.
+
+Three trigger paths
+~~~~~~~~~~~~~~~~~~~
+
+- **``datasette serve`` (CLI)** — startup and ``uvicorn.Server.serve()`` both run inside a single ``asyncio.run()`` call, so there is exactly one event loop for the whole life of the process.
+- **ASGI lifespan** — ``Datasette.app()`` wires startup and background-task launch into the ``on_startup`` list, and shutdown into the ``on_shutdown`` list, of an internal ``AsgiLifespan`` wrapper. A spec-compliant ASGI server (uvicorn, hypercorn, and others) sends the ``lifespan.startup`` message and waits for ``lifespan.startup.complete`` before delivering any ``http`` or ``websocket`` scope, so startup — including every plugin's own internal-database migrations — is guaranteed to have finished before any request reaches Datasette, including requests seen by plugin :ref:`asgi_wrapper <plugin_asgi_wrapper>` middleware. If a ``startup`` hook raises, ``AsgiLifespan`` sends ``lifespan.startup.failed`` with the exception message instead of hanging or crashing ambiguously, so the host can abort the boot cleanly.
+- **First-request fallback** — an internal ``AsgiRunOnFirstRequest`` wrapper runs the same startup work as a safety net for hosts that never send ASGI lifespan events at all: some ASGI mounts, a bare ``app()`` embedded inside another framework, and :ref:`datasette.client <internals_datasette_client>` / test clients, which drive requests directly over ``httpx2.ASGITransport`` without ever emitting ``lifespan.startup``. It runs startup exactly once, the first time any non-lifespan scope arrives, guarded by a lock so that concurrent early requests can't run it twice.
+
+All three paths call the same idempotent internal methods, so it is safe for more than one of them to fire — lifespan startup completing and then a first request arriving afterwards is a no-op the second time. A host that never sends lifespan events and never goes through the CLI degrades to first-request timing: startup runs on the first request instead of before it, exactly as Datasette always worked prior to this lifecycle guarantee. This is a deliberate fallback rather than a regression — see :ref:`datasette_add_background_task` for how to opt out of launching background tasks (the ``--get`` CLI path) or drive startup and launch explicitly (tests, headless embedders).
+
+.. _datasette_add_background_task:
+
+.add_background_task(func, name=None)
+-------------------------------------
+
+``func`` - async callable
+    A coroutine function taking one positional argument, the ``Datasette`` instance. Core calls ``await func(datasette)``.
+
+``name`` - string, optional
+    A name for the task, used to identify it in the ``/-/tasks`` introspection endpoint (:ref:`JsonDataView_tasks`) and in log messages. Defaults to ``func.__qualname__``. If the resulting name collides with an already-registered task, a ``-2``, ``-3``, ... suffix is appended.
+
+Registers a piece of supervised, long-lived background work — typically called from a :ref:`plugin_hook_startup` hook, though it can be called at any point after the instance exists, including from a request handler. Returns a :ref:`BackgroundTask <BackgroundTask>` handle.
+
+Registration is separate from launch. Calling this from a ``startup`` hook — the common case — buffers the task; core launches every registered task once *all* ``startup`` hooks have completed, as described in :ref:`datasette_lifecycle`. Calling it after launch has already happened — for example from a request handler, to start a per-job task dynamically — starts the task immediately instead.
+
+.. code-block:: python
+
+    import asyncio
+    from datasette import hookimpl
+
+
+    async def poll_for_updates(datasette):
+        while True:
+            await do_one_poll(datasette)
+            await asyncio.sleep(60)
+
+
+    @hookimpl
+    def startup(datasette):
+        datasette.add_background_task(
+            poll_for_updates, name="my-plugin-poller"
+        )
+
+Core owns the task for the rest of the process's life:
+
+- **A strong reference is kept forever**, so the task can never be silently garbage collected the way an unreferenced ``asyncio.create_task()`` call can be.
+- **A crash is logged, not swallowed.** If ``func`` raises anything other than ``asyncio.CancelledError``, the exception (with its traceback) is logged to the ``datasette.background_tasks`` logger and recorded on the handle's ``.exception``, and the task's ``.state`` becomes ``crashed``.
+- **Cancellation is coordinated.** On shutdown, every task that is still running is cancelled and given a grace period to stop — see :ref:`datasette_lifecycle`.
+
+Launch matrix
+~~~~~~~~~~~~~
+
+Whether registered tasks actually launch depends on how the instance is being run:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Trigger
+     - Launches registered tasks?
+   * - ASGI lifespan (real server deployments)
+     - Yes, after ``lifespan.startup`` completes
+   * - First-request fallback (lifespan-less hosts)
+     - Yes, on the first request — parity with the lifespan case
+   * - ``datasette serve --get``
+     - Never
+   * - Tests / headless embedders
+     - Only if you call :ref:`datasette_start_background_tasks` explicitly
+
+``datasette --get`` never launches background tasks, even though its one-shot request flows through the same first-request fallback as everything else: it sets an internal flag before making that request specifically to suppress the launch, since a one-shot CLI invocation has no server loop left running afterwards to keep any launched tasks alive.
+
+.. _BackgroundTask:
+
+BackgroundTask objects
+~~~~~~~~~~~~~~~~~~~~~~
+
+``add_background_task()`` returns a ``BackgroundTask`` handle with the following attributes:
+
+``.name`` - string
+    The task's (unique) name.
+
+``.state`` - string
+    One of ``registered`` (added but not yet launched), ``running``, ``completed`` (returned cleanly), ``crashed`` (raised an exception) or ``cancelled``.
+
+``.task`` - ``asyncio.Task`` or ``None``
+    The underlying ``asyncio.Task``, once launched. ``None`` while still ``registered``.
+
+``.exception`` - ``BaseException`` or ``None``
+    The exception that crashed the task, if ``.state`` is ``crashed``.
+
+``.started_at`` - string or ``None``
+    ISO 8601 UTC timestamp of when the task was launched.
+
+``.function`` - string
+    The callable's dotted module and qualified name, for example ``my_plugin.jobs.poll_for_updates``.
+
+``.cancel()``
+    Cancel the task. If it has already launched, this cancels the underlying ``asyncio.Task`` — ``.state`` becomes ``cancelled`` once the cancellation is observed. If it has not launched yet, it is removed from the queue so it never runs.
+
+This is also the shape of each entry returned by the ``/-/tasks`` JSON introspection endpoint — see :ref:`JsonDataView_tasks`.
+
+.. _datasette_start_background_tasks:
+
+await .start_background_tasks()
+-------------------------------
+
+Runs startup (if it has not already run) and launches every task registered with :ref:`datasette_add_background_task`. This is the explicit equivalent of what happens automatically via ASGI lifespan or the first-request fallback in a served deployment — the entry point for tests and headless embedders (a cron-style CLI command that wants supervised background work without running a server) that need background tasks without going through either of those paths.
+
+.. code-block:: python
+
+    datasette = Datasette(memory=True)
+    await datasette.start_background_tasks()
+
+.. note::
+
+   ``start_background_tasks()`` calls ``invoke_startup()`` internally, **not** the fuller startup sequence a served instance uses — so calling it directly, without a prior request through ``datasette.client``, skips the immutable-database table-count precompute that a real server performs as part of startup. This only matters if your code inspects table counts before any request has been made; if you also exercise the instance via ``datasette.client`` (which arms the first-request fallback, and therefore the full startup sequence including table counts), or don't care about table counts up front, there is nothing to worry about.
 
 .. _datasette_track_event:
 
@@ -1594,32 +1726,32 @@ datasette.client
 
 Plugins can make internal simulated HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins, while avoiding the overhead of making an external HTTP call to access those APIs.
 
-The ``datasette.client`` object is a wrapper around the `HTTPX Python library <https://www.python-httpx.org/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
+The ``datasette.client`` object is a wrapper around the `HTTPX2 Python library <https://httpx2.pydantic.dev/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
 
 It offers the following methods:
 
-``await datasette.client.get(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.get(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal GET request against that path.
 
-``await datasette.client.post(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.post(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal POST request. Use ``data={"name": "value"}`` to pass form parameters.
 
-``await datasette.client.options(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.options(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal OPTIONS request.
 
-``await datasette.client.head(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.head(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal HEAD request.
 
-``await datasette.client.put(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.put(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal PUT request.
 
-``await datasette.client.patch(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.patch(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal PATCH request.
 
-``await datasette.client.delete(path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.delete(path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal DELETE request.
 
-``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX Response
+``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX2 Response
     Execute an internal request with the given HTTP method against that path.
 
 These methods can be used with :ref:`internals_datasette_urls` - for example:
@@ -1636,7 +1768,7 @@ These methods can be used with :ref:`internals_datasette_urls` - for example:
 
 ``datasette.client`` methods automatically take the current :ref:`setting_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path.
 
-For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation <https://www.python-httpx.org/async/>`__.
+For documentation on available ``**kwargs`` options and the shape of the HTTPX2 Response object refer to the `HTTPX2 Async documentation <https://httpx2.pydantic.dev/async/>`__.
 
 .. _internals_datasette_client_actor:
 
@@ -2023,8 +2155,8 @@ Example usage:
 
 .. _database_execute_write:
 
-await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True)
+await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True, time_limit_ms=2000)
---------------------------------------------------------------------------------------------------------------------------
+----------------------------------------------------------------------------------------------------------------------------------------------
 
 SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received.
 
@@ -2063,6 +2195,13 @@ Each call to ``execute_write()`` will be executed inside a transaction. Pass
 ``transaction=False`` for statements such as ``VACUUM`` that cannot run inside
 a transaction.
 
+Write statements have a default time limit of 2,000ms. Pass a different value
+using ``time_limit_ms=`` or use ``time_limit_ms=None`` to allow the statement to
+run without a time limit.
+
+This write limit is independent of the ``sql_time_limit_ms`` setting used for
+read queries. Changing that setting does not change the default write limit.
+
 .. _database_execute_write_script:
 
 await db.execute_write_script(sql, block=True)
@@ -2623,12 +2762,12 @@ This example uses trace to record the start, end and duration of any HTTP GET re
 .. code-block:: python
 
     from datasette.tracer import trace
-    import httpx
+    import httpx2
 
 
     async def fetch_url(url):
         with trace("fetch-url", url=url):
-            async with httpx.AsyncClient() as client:
+            async with httpx2.AsyncClient() as client:
                 return await client.get(url)
 
 .. _internals_tracer_trace_child_tasks:

docs/introspection.rst

@@ -9,7 +9,7 @@ Each of these pages can be viewed in your browser. Add ``.json`` to the URL to g
 
 JSON responses that return an object include an ``"ok": true`` key, consistent with the rest of the :ref:`JSON API <json_api>`.
 
-The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise <json_api_stability>`, with the exception of the debug endpoints ``/-/threads`` and ``/-/actions``, whose shapes may change in future releases.
+The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise <json_api_stability>`, with the exception of the debug endpoints ``/-/threads``, ``/-/tasks`` and ``/-/actions``, whose shapes may change in future releases.
 
 .. _JsonDataView_metadata:
 
@@ -278,6 +278,42 @@ Shows details of threads and ``asyncio`` tasks. This endpoint requires the ``per
         ]
     }
 
+.. _JsonDataView_tasks:
+
+/-/tasks
+--------
+
+Shows the state of every supervised background task registered with :ref:`datasette.add_background_task() <datasette_add_background_task>`; see also :ref:`BackgroundTask <BackgroundTask>` for what each field below means, and :ref:`datasette_lifecycle` for when tasks are launched. This endpoint requires the ``permissions-debug`` permission, since a crashed task's ``exception`` field can reveal internals such as file paths or query text:
+
+.. code-block:: json
+
+    {
+        "ok": true,
+        "tasks": [
+            {
+                "name": "my_plugin.poll_for_updates",
+                "state": "running",
+                "function": "my_plugin.poll_for_updates",
+                "started_at": "2026-07-30T12:00:00+00:00",
+                "exception": null
+            },
+            {
+                "name": "my_plugin.broken_task",
+                "state": "crashed",
+                "function": "my_plugin.broken_task",
+                "started_at": "2026-07-30T12:00:00+00:00",
+                "exception": "ValueError('something went wrong')"
+            }
+        ],
+        "launched": true
+    }
+
+Each entry's ``function`` identifies the callable by its dotted module and qualified name.
+
+Each entry's ``state`` is one of ``registered`` (added but not yet launched), ``running``, ``completed``, ``crashed`` or ``cancelled``. ``exception`` is a one-line ``repr()`` of the exception for a ``crashed`` task, or ``null`` otherwise - the full traceback is written to the ``datasette.background_tasks`` logger instead, to keep this payload skimmable.
+
+The top-level ``launched`` flag reports whether the instance has run its one-time background task launch (after ``startup`` hooks finish, or via lifespan/first-request/:ref:`start_background_tasks() <datasette_start_background_tasks>`). It distinguishes "no tasks have been registered" (``tasks`` is empty either way) from "tasks are registered but nothing has armed the launch yet" (``launched`` is ``false`` and every task's ``state`` is still ``registered``) - useful when debugging a host that never triggers Datasette's lifespan events.
+
 .. _JsonDataView_actor:
 
 /-/actor

docs/json_api.rst

@@ -48,6 +48,7 @@ Some JSON endpoints are **exempt** from this promise:
   debug playground.
 - Debug and support endpoints are documented so you can use them, but their
   JSON shapes are not frozen: :ref:`/-/threads <JsonDataView_threads>`,
+  :ref:`/-/tasks <JsonDataView_tasks>`,
   :ref:`/-/actions <JsonDataView_actions>`,
   the :ref:`permission debug endpoints <PermissionsDebugView>`
   (``/-/allowed``, ``/-/rules``, ``/-/check``) and the
@@ -1326,6 +1327,23 @@ The following extras are available for arbitrary SQL query responses and stored,
 
 .. [[[end]]]
 
+.. _TableCountView:
+
+Counting all matching rows
+--------------------------
+
+``POST /<database>/<table>/-/count`` returns an exact count of the rows matching the table's query string filters::
+
+    POST /fixtures/facetable/-/count?state=CA
+
+    {"ok": true, "count": 10}
+
+The endpoint supports the same column, search and plugin filters as the table page. Pagination and display options such as ``_next``, ``_size`` and ``_sort`` do not affect the count.
+
+This requires ``view-table`` permission. ``execute-sql`` permission is only needed if using ``_where`` filters.
+
+Unlike the ``count`` extra, this count is not capped by the row count limit. The usual SQL time limit still applies; a timed-out count returns a 400 JSON error.
+
 .. _TableAutocompleteView:
 
 Table autocomplete
@@ -1661,6 +1679,8 @@ The request body is always parsed as JSON, regardless of the request's ``Content
 
 The row-based write APIs can write :ref:`binary values in JSON <binary_json_format>` using Datasette's Base64 representation for BLOB data.
 
+Structured inserts, upserts, updates and deletes only support ordinary SQLite tables. Virtual tables and their internal shadow tables are rejected, including when adding rows to an existing table through the create-table API. Writes to ordinary content tables can still update full-text search indexes through configured triggers.
+
 .. _ExecuteWriteView:
 
 Executing write SQL

docs/plugin_hooks.rst

@@ -1107,7 +1107,7 @@ Return an `ASGI <https://asgi.readthedocs.io/>`__ middleware wrapper function th
 
 This is a very powerful hook. You can use it to manipulate the entire Datasette response, or even to configure new URL routes that will be handled by your own custom code.
 
-You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette <https://www.starlette.io/middleware/>`__.
+You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette <https://starlette.dev/middleware/>`__.
 
 This example plugin adds a ``x-databases`` HTTP header listing the currently attached databases:
 
@@ -1157,7 +1157,7 @@ Examples: `datasette-cors <https://datasette.io/plugins/datasette-cors>`__, `dat
 startup(datasette)
 ------------------
 
-This hook fires when the Datasette application server first starts up.
+This hook fires when the Datasette application server first starts up. It runs on the same event loop that goes on to serve requests, so it is safe to create loop-bound primitives and register background work here — see :ref:`datasette_lifecycle` for the full guarantee and the three ways startup can be triggered.
 
 Here is an example that validates required plugin configuration. The server will fail to start and show an error if the validation check fails:
 
@@ -1195,6 +1195,7 @@ Potential use-cases:
 * Create database tables that a plugin needs on startup
 * Validate the configuration for a plugin on startup, and raise an error if it is invalid
 * Raise a ``datasette.utils.StartupError("message")`` exception to prevent Datasette from starting and display that message to the user.
+* Register supervised long-lived background work using :ref:`datasette_add_background_task`, which core launches once every plugin's ``startup()`` hook has finished.
 
 .. note::
 
@@ -1211,6 +1212,31 @@ Potential use-cases:
 
 Examples: `datasette-saved-queries <https://datasette.io/plugins/datasette-saved-queries>`__, `datasette-init <https://datasette.io/plugins/datasette-init>`__
 
+.. _plugin_hook_shutdown:
+
+shutdown(datasette)
+-------------------
+
+This hook fires once, when the Datasette application server is shutting down gracefully - triggered by the ASGI ``lifespan.shutdown`` event, which includes pressing Ctrl-C or sending ``SIGTERM`` to a ``datasette serve`` process. It is not called on a hard kill (``SIGKILL``), since there is no opportunity to run any code in that case.
+
+Like ``startup()``, this can be a regular function or it can return an async function to be awaited.
+
+It runs before Datasette cancels any background tasks it is supervising (see :ref:`datasette_add_background_task`) and before it closes its database connections, so you can use it to tell your plugin's own background work to stop gracefully while a database connection is still available to write out any final state. See :ref:`datasette_lifecycle` for exactly where this fits into the full startup-to-shutdown sequence:
+
+.. code-block:: python
+
+    @hookimpl
+    def shutdown(datasette):
+        async def inner():
+            db = datasette.get_database()
+            await db.execute_write(
+                "insert into shutdown_log (at) values (datetime('now'))"
+            )
+
+        return inner
+
+If your ``shutdown()`` hook raises an exception it will be logged but not re-raised, so one plugin's broken shutdown code cannot prevent other plugins - or Datasette itself - from finishing their own teardown.
+
 .. _plugin_hook_actor_from_request:
 
 actor_from_request(datasette, request)

docs/plugins.rst

@@ -261,6 +261,15 @@ If you run ``datasette plugins --all`` it will include default plugins that ship
                 "permission_resources_sql"
             ]
         },
+        {
+            "name": "datasette.default_permissions.sqlite_statistics",
+            "static": false,
+            "templates": false,
+            "version": null,
+            "hooks": [
+                "permission_resources_sql"
+            ]
+        },
         {
             "name": "datasette.default_permissions.tokens",
             "static": false,

docs/settings.rst

@@ -71,6 +71,8 @@ Should users be able to execute arbitrary SQL queries by default?
 
 Setting this to ``off`` causes permission checks for :ref:`actions_execute_sql` to fail by default.
 
+This setting controls the ability to submit arbitrary SQL. It does not disable structured table-browsing features that use SQL generated by Datasette, such as sorting, column filters and :ref:`facets`. Use :ref:`setting_allow_facet` to control whether users can request facets.
+
 ::
 
     datasette mydatabase.db --setting default_allow_sql off
@@ -254,6 +256,8 @@ Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-ag
 
     datasette mydatabase.db --setting default_cache_ttl 60
 
+Dynamic responses for authenticated actors, requests with cookies or an ``Authorization`` header, and responses that set cookies use ``Cache-Control: private, no-store``. This takes precedence over ``default_cache_ttl`` and ``?_ttl=``, even when cache headers are otherwise disabled. Anonymous dynamic responses vary by ``Cookie`` and ``Authorization``. Static assets retain their own cache policy.
+
 .. _setting_cache_size_kb:
 
 cache_size_kb

docs/testing_plugins.rst

@@ -25,7 +25,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl
         )
 
 
-This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX <https://www.python-httpx.org/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
+This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX2 <https://httpx2.pydantic.dev/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
 
 This test also uses the `pytest-asyncio <https://pypi.org/project/pytest-asyncio/>`__ package to add support for ``async def`` test functions running under pytest.
 
@@ -78,9 +78,19 @@ Creating a ``Datasette()`` instance like this as useful shortcut in tests, but t
     datasette = Datasette(memory=True)
     await datasette.invoke_startup()
 
-This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls.
+This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls. It runs on the same event loop that runs your test, matching the guarantee described in :ref:`datasette_lifecycle`.
 
-If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request.
+If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request, via the first-request fallback described in :ref:`datasette_lifecycle`.
+
+If your plugin also registers work with :ref:`datasette_add_background_task` (typically from a ``startup`` hook) and your test needs that work to actually run, call ``await datasette.start_background_tasks()`` as well - ``invoke_startup()`` alone only runs ``startup`` hooks, it does not launch anything they registered:
+
+.. code-block:: python
+
+    datasette = Datasette(memory=True)
+    await datasette.start_background_tasks()
+    # Any tasks registered by a startup() hook are now running
+
+A request made through ``datasette.client`` arms both startup and background-task launch automatically, since they're both part of the same first-request fallback - ``start_background_tasks()`` is for tests that need tasks running without making an HTTP request first.
 
 .. _testing_plugins_datasette_fixtures_database:
 
@@ -154,7 +164,7 @@ If you need to opt out of this behavior, add the following to your ``pytest.ini`
 Using datasette.client in tests
 -------------------------------
 
-The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client <https://www.python-httpx.org/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
+The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX2 async client <https://httpx2.pydantic.dev/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
 
 A simple test looks like this:
 
@@ -273,22 +283,22 @@ If you want to create that test database repeatedly for every individual test fu
 
 .. _testing_plugins_pytest_httpx:
 
-Testing outbound HTTP calls with pytest-httpx
+Testing outbound HTTP calls with pytest-httpx2
----------------------------------------------
+----------------------------------------------
 
 If your plugin makes outbound HTTP calls - for example datasette-auth-github or datasette-import-table - you may need to mock those HTTP requests in your tests.
 
-The `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__ package is a useful library for mocking calls. It can be tricky to use with Datasette though since it mocks all HTTPX requests, and Datasette's own testing mechanism uses HTTPX internally.
+The `pytest-httpx2 <https://pypi.org/project/pytest-httpx2/>`__ package provides a ``httpx2_mock`` fixture, built on `respx <https://lundberg.github.io/respx/>`__, for mocking outbound calls made using HTTPX2.
 
-To avoid breaking your tests, you can return ``["localhost"]`` from the ``non_mocked_hosts()`` fixture.
+Datasette's own ``datasette.client`` mechanism uses HTTPX2 internally too, but those requests are passed directly to the ASGI application rather than being sent over the network, so they are not affected by the mock.
 
-As an example, here's a very simple plugin which executes an HTTP response and returns the resulting content:
+As an example, here's a very simple plugin which executes an HTTP request and returns the resulting content:
 
 .. code-block:: python
 
     from datasette import hookimpl
     from datasette.utils.asgi import Response
-    import httpx
+    import httpx2
 
 
     @hookimpl
@@ -306,27 +316,18 @@ As an example, here's a very simple plugin which executes an HTTP response and r
             </form>""")
         vars = await request.post_vars()
         url = vars["url"]
-        return Response.text(httpx.get(url).text)
+        return Response.text(httpx2.get(url).text)
 
-Here's a test for that plugin that mocks the HTTPX outbound request:
+Here's a test for that plugin that mocks the HTTPX2 outbound request:
 
 .. code-block:: python
 
     from datasette.app import Datasette
-    import pytest
 
 
-    @pytest.fixture
+    async def test_outbound_http_call(httpx2_mock):
-    def non_mocked_hosts():
+        httpx2_mock.get("https://www.example.com/").respond(
-        # This ensures httpx-mock will not affect Datasette's own
+            text="Hello world"
-        # httpx calls made in the tests by datasette.client:
-        return ["localhost"]
-
-
-    async def test_outbound_http_call(httpx_mock):
-        httpx_mock.add_response(
-            url="https://www.example.com/",
-            text="Hello world",
         )
         datasette = Datasette([], memory=True)
         response = await datasette.client.post(
@@ -335,11 +336,13 @@ Here's a test for that plugin that mocks the HTTPX outbound request:
         )
         assert response.text == "Hello world"
 
-        outbound_request = httpx_mock.get_request()
+        outbound_request = httpx2_mock.calls.last.request
         assert (
             outbound_request.url == "https://www.example.com/"
         )
 
+If your plugin still makes its outbound calls using the original ``httpx`` library you can continue to mock those using `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__.
+
 .. _testing_plugins_register_in_test:
 
 Registering a plugin for the duration of a test

pyproject.toml

@@ -28,9 +28,9 @@ dependencies = [
     "click-default-group>=1.2.3",
     "Jinja2>=2.10.3",
     "hupper>=1.9",
-    "httpx>=0.20,<1.0",
+    "httpx2>=2.0",
     "pluggy>=1.0",
-    "uvicorn>=0.11",
+    "uvicorn>=0.29",
     "aiofiles>=0.4",
     "PyYAML>=5.3",
     "mergedeep>=1.1.1",

tests/conftest.py

@@ -2,13 +2,14 @@ import importlib.metadata
 import os
 import pathlib
 import re
+import socket
 import subprocess
 import sys
 import tempfile
 import time
 from dataclasses import dataclass
 
-import httpx
+import httpx2
 import pytest
 import pytest_asyncio
 
@@ -32,17 +33,31 @@ UNDOCUMENTED_PERMISSIONS = {
 }
 
 
-def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs):
+def wait_until_responds(url, timeout=5.0, client=httpx2, process=None, **kwargs):
     start = time.time()
     while time.time() - start < timeout:
+        # If the server died there is no point waiting out the timeout - fail
+        # now, with its output, instead of after `timeout` seconds of silence
+        if process is not None and process.poll() is not None:
+            raise AssertionError(
+                "Server exited early with returncode {}\n{}".format(
+                    process.returncode, process.stdout.read().decode("utf-8")
+                )
+            )
         try:
             client.get(url, **kwargs)
             return
-        except httpx.ConnectError:
+        except httpx2.TransportError:
             time.sleep(0.1)
     raise AssertionError(f"Timed out waiting for {url} to respond")
 
 
+def find_free_port():
+    with socket.socket() as sock:
+        sock.bind(("127.0.0.1", 0))
+        return sock.getsockname()[1]
+
+
 @pytest.fixture
 def bare_ds():
     """
@@ -277,8 +292,8 @@ def ds_unix_domain_socket_server(tmp_path_factory):
         cwd=tempfile.gettempdir(),
     )
     # Poll until available
-    transport = httpx.HTTPTransport(uds=uds)
+    transport = httpx2.HTTPTransport(uds=uds)
-    client = httpx.Client(transport=transport)
+    client = httpx2.Client(transport=transport)
     try:
         wait_until_responds(
             "http://localhost/_memory.json", timeout=30.0, client=client
@@ -301,6 +316,71 @@ def ds_unix_domain_socket_server(tmp_path_factory):
             pass
 
 
+@pytest.fixture
+def serve_with_plugins(tmp_path):
+    """Factory fixture for starting ``datasette serve`` in a subprocess with
+    plugins written to a temporary ``--plugins-dir``.
+
+    For tests that need the real serve path: event-loop wiring, exit codes,
+    signals. The usual in-process ``pm.register`` plugin pattern can't reach
+    a subprocess, so plugin source is written out as importable files instead.
+
+    Unlike ``ds_localhost_http_server`` this is function-scoped and takes a
+    fresh port each time, because each test needs its own plugins. Call it as::
+
+        proc, port = serve_with_plugins({"my_plugin": PLUGIN_SOURCE})
+
+    ``plugins`` maps module name to Python source. Pass
+    ``wait_for_startup=False`` when the server is expected to fail during
+    startup rather than begin serving. Extra CLI arguments are passed through.
+    Every process started is terminated when the test ends.
+    """
+    processes = []
+
+    def start(plugins, *extra_args, wait_for_startup=True):
+        plugins_dir = tmp_path / "plugins"
+        plugins_dir.mkdir(exist_ok=True)
+        for module_name, source in plugins.items():
+            (plugins_dir / f"{module_name}.py").write_text(source, "utf-8")
+        port = find_free_port()
+        proc = subprocess.Popen(
+            [
+                sys.executable,
+                "-m",
+                "datasette",
+                "--memory",
+                "--plugins-dir",
+                str(plugins_dir),
+                "-h",
+                "127.0.0.1",
+                "-p",
+                str(port),
+                *extra_args,
+            ],
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            # Avoid FileNotFoundError: [Errno 2] No such file or directory:
+            cwd=tempfile.gettempdir(),
+        )
+        processes.append(proc)
+        if wait_for_startup:
+            wait_until_responds(
+                f"http://127.0.0.1:{port}/-/versions.json", process=proc
+            )
+        return proc, port
+
+    yield start
+
+    for proc in processes:
+        if proc.poll() is None:
+            proc.terminate()
+            try:
+                proc.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                proc.kill()
+                proc.wait()
+
+
 # Import fixtures from fixtures.py to make them available
 from .fixtures import (  # noqa: F401
     TEMP_PLUGIN_SECRET_FILE,

tests/test_api.py

@@ -5,6 +5,7 @@ import pytest
 
 from datasette.app import Datasette
 from datasette.plugins import DEFAULT_PLUGINS
+from datasette.resources import DatabaseResource, TableResource
 from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode
 from datasette.utils.sqlite import sqlite_version
 from datasette.version import __version__
@@ -101,14 +102,11 @@ async def test_database_page(ds_client):
         "tags",
     }
 
-    # Expected hidden tables
+    # The external-content index is visible, but its shadow tables need a
+    # second dependency hop and are excluded by the one-hop permission policy.
     expected_hidden_tables = {
         "no_primary_key",
         "searchable_fts",
-        "searchable_fts_config",
-        "searchable_fts_data",
-        "searchable_fts_docsize",
-        "searchable_fts_idx",
     }
 
     # Verify all expected tables exist
@@ -458,6 +456,67 @@ async def test_row_foreign_key_tables(ds_client):
     ]
 
 
+@pytest.mark.asyncio
+async def test_row_foreign_key_tables_omit_denied_tables(request):
+    actor = {"id": "reader"}
+    ds = Datasette(
+        memory=True,
+        default_deny=True,
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "parents": {"permissions": {"view-table": True}},
+                        "private_children": {"permissions": {"view-table": False}},
+                    }
+                }
+            }
+        },
+    )
+    request.addfinalizer(ds.close)
+    db = ds.add_memory_database("fk_count_leak", name="data")
+    await db.execute_write("create table parents (id integer primary key, name text)")
+    await db.execute_write("""
+        create table private_children (
+            id integer primary key,
+            parent_id integer references parents(id)
+        )
+    """)
+    await db.execute_write("insert into parents values (1, 'Public parent')")
+    await db.execute_write("""
+        insert into private_children (id, parent_id) values
+            (1, 1),
+            (2, 1),
+            (3, 1)
+    """)
+    await ds.invoke_startup()
+
+    parent = TableResource(database="data", table="parents")
+    private_children = TableResource(database="data", table="private_children")
+    assert await ds.allowed(action="view-table", resource=parent, actor=actor)
+    assert not await ds.allowed(
+        action="view-table", resource=private_children, actor=actor
+    )
+    assert not await ds.allowed(
+        action="execute-sql",
+        resource=DatabaseResource(database="data"),
+        actor=actor,
+    )
+
+    direct_child = await ds.client.get("/data/private_children.json", actor=actor)
+    assert direct_child.status_code == 403
+    parent_response = await ds.client.get(
+        "/data/parents/1.json?_extra=foreign_key_tables", actor=actor
+    )
+    assert parent_response.status_code == 200
+
+    foreign_key_tables = parent_response.json().get("foreign_key_tables", [])
+    assert foreign_key_tables == [], (
+        "denied child table name, foreign-key column, and row count disclosed: "
+        f"{foreign_key_tables}"
+    )
+
+
 @pytest.mark.asyncio
 async def test_row_extras(ds_client):
     response = await ds_client.get(
@@ -894,10 +953,7 @@ async def test_hidden_sqlite_stat1_table():
     await db.execute_write("analyze")
     data = (await ds.client.get("/db.json?_show_hidden=1")).json()
     tables = [(t["name"], t["hidden"]) for t in data["tables"]]
-    assert tables in (
+    assert tables == [("normal", False)]
-        [("normal", False), ("sqlite_stat1", True)],
-        [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)],
-    )
 
 
 @pytest.mark.asyncio

tests/test_api_write.py

@@ -1,6 +1,7 @@
 import time
 
 import pytest
+import sqlite_utils
 
 from datasette.app import Datasette
 from datasette.events import RenameTableEvent
@@ -67,6 +68,82 @@ BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"}
 BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}'
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("use_fallback", (False, True))
+@pytest.mark.parametrize(
+    "operation", ("insert", "upsert", "update", "delete", "create", "create_uppercase")
+)
+@pytest.mark.parametrize(
+    "module,definition,values,shadow_suffix",
+    (
+        ("fts5", "body", "'original'", "_content"),
+        ("fts4", "body", "'original'", "_content"),
+        ("rtree", "id, minx, maxx", "1, 0, 1", "_rowid"),
+    ),
+)
+@pytest.mark.parametrize("shadow", (False, True))
+async def test_structured_writes_require_ordinary_tables(
+    ds_write,
+    monkeypatch,
+    use_fallback,
+    operation,
+    module,
+    definition,
+    values,
+    shadow_suffix,
+    shadow,
+):
+    if use_fallback:
+        monkeypatch.setattr("datasette.utils.sqlite.supports_table_list", lambda: False)
+    db = ds_write.get_database("data")
+    await db.execute_write(f"create virtual table indexed using {module}({definition})")
+    await db.execute_write(f"insert into indexed values ({values})")
+    table = "indexed" + (shadow_suffix if shadow else "")
+    row = (await db.execute(f"select rowid, * from {escape_sqlite(table)}")).dicts()[0]
+    pks = await db.primary_keys(table)
+    pk_value = row[pks[0] if pks else "rowid"]
+    before = await db.execute_fn(lambda conn: list(conn.iterdump()))
+
+    if operation in ("create", "create_uppercase"):
+        path = "/data/-/create"
+        body = {
+            "table": table.upper() if operation == "create_uppercase" else table,
+            "rows": [row],
+        }
+    elif operation in ("update", "delete"):
+        path = f"/data/{table}/{pk_value}/-/{operation}"
+        body = {"update": row} if operation == "update" else {}
+    else:
+        path = f"/data/{table}/-/{operation}"
+        body = {"rows": [row]}
+    response = await ds_write.client.post(
+        path, json=body, headers=_headers(write_token(ds_write))
+    )
+    assert response.status_code == 400, response.text
+    assert response.json()["errors"] == ["Structured writes require an ordinary table"]
+    assert await db.execute_fn(lambda conn: list(conn.iterdump())) == before
+
+
+@pytest.mark.asyncio
+async def test_structured_writes_to_content_table_maintain_fts(ds_write):
+    db = ds_write.get_database("data")
+    await db.execute_write_fn(
+        lambda conn: sqlite_utils.Database(conn)["docs"].enable_fts(
+            ["title"], create_triggers=True
+        )
+    )
+    response = await ds_write.client.post(
+        "/data/docs/-/insert",
+        json={"row": {"id": 1, "title": "ordinary content"}},
+        headers=_headers(write_token(ds_write)),
+    )
+    assert response.status_code == 201, response.text
+    matches = await db.execute(
+        "select rowid from docs_fts where docs_fts match ?", ["ordinary"]
+    )
+    assert [row[0] for row in matches.rows] == [1]
+
+
 @pytest.mark.asyncio
 async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write):
     token = write_token(ds_write)
@@ -1295,7 +1372,7 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w
 
 @pytest.mark.asyncio
 async def test_foreign_key_suggestions(ds_write):
-    token = write_token(ds_write, permissions=["at"])
+    token = write_token(ds_write, permissions=["alter-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
     await db.execute_write("insert into owners (id) values (1), (2), (3)")
@@ -1361,7 +1438,7 @@ async def test_foreign_key_suggestions_permission_denied(ds_write):
 
 @pytest.mark.asyncio
 async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch):
-    token = write_token(ds_write, permissions=["at"])
+    token = write_token(ds_write, permissions=["alter-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
 
@@ -1392,7 +1469,7 @@ async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch):
 
 @pytest.mark.asyncio
 async def test_foreign_key_targets(ds_write):
-    token = write_token(ds_write, permissions=["ct"])
+    token = write_token(ds_write, permissions=["create-table", "view-table"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
     await db.execute_write("create table categories (slug varchar(30) primary key)")
@@ -1725,6 +1802,42 @@ async def test_drop_table(ds_write, scenario):
         assert (await ds_write.client.get("/data/docs")).status_code == 404
 
 
+@pytest.mark.asyncio
+async def test_drop_table_cleans_up_fts(ds_write):
+    db = ds_write.get_database("data")
+
+    def enable_fts(conn):
+        sqlite_utils.Database(conn)["docs"].enable_fts(["title"], create_triggers=True)
+
+    await db.execute_write_fn(enable_fts)
+    assert {
+        row[0]
+        for row in await db.execute(
+            "select name from sqlite_master where type = 'table' and name like 'docs_fts%'"
+        )
+    } == {
+        "docs_fts",
+        "docs_fts_config",
+        "docs_fts_data",
+        "docs_fts_docsize",
+        "docs_fts_idx",
+    }
+
+    response = await ds_write.client.post(
+        "/data/docs/-/drop",
+        json={"confirm": True},
+        headers=_headers(write_token(ds_write)),
+    )
+
+    assert response.json() == {"ok": True}
+    assert [
+        row[0]
+        for row in await db.execute(
+            "select name from sqlite_master where type = 'table' and name like 'docs_fts%'"
+        )
+    ] == []
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "input,expected_status,expected_response,expected_events",
@@ -2708,3 +2821,119 @@ async def test_create_using_alter_against_existing_table(
         insert_rows_event = ds_write._tracked_events[1]
         assert insert_rows_event.name == "insert-rows"
         assert insert_rows_event.num_rows == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("denied_action", "request_body"),
+    (
+        (
+            "insert-row",
+            {
+                "table": "salaries",
+                "rows": [{"id": 9, "note": "INJ-VIA-CREATE"}],
+            },
+        ),
+        (
+            "update-row",
+            {
+                "table": "salaries",
+                "rows": [{"id": 1, "note": "REPLACED"}],
+                "pk": "id",
+                "replace": True,
+            },
+        ),
+        (
+            "alter-table",
+            {
+                "table": "salaries",
+                "rows": [{"id": 9, "note": "INSERTED", "extra": "NEW"}],
+                "alter": True,
+            },
+        ),
+    ),
+)
+async def test_create_table_existing_table_respects_table_level_denial(
+    denied_action, request_body
+):
+    # GHSA-53fc-rhfg-h7qp issue 2: POST /db/-/create against an existing table
+    # inserts rows into it, so insert-row (and update-row / alter-table) must be
+    # checked against the TableResource, not just the DatabaseResource.
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                # id=editor user has each permission at the database level, but
+                # the selected action is explicitly denied on the salaries table
+                "data": {
+                    "permissions": {
+                        "create-table": {"id": "editor"},
+                        "insert-row": {"id": "editor"},
+                        "update-row": {"id": "editor"},
+                        "alter-table": {"id": "editor"},
+                    },
+                    "tables": {
+                        "salaries": {"permissions": {denied_action: False}},
+                    },
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database(
+        f"create_table_existing_table_denied_{denied_action}", name="data"
+    )
+    await db.execute_write("create table salaries (id integer primary key, note text)")
+    await db.execute_write("insert into salaries values (1, 'TOPSECRET-A')")
+    await ds.invoke_startup()
+
+    if denied_action == "insert-row":
+        # Sanity: direct insert into salaries is denied for this actor
+        direct = await ds.client.post(
+            "/data/salaries/-/insert",
+            actor={"id": "editor"},
+            json={"row": {"id": 9, "note": "INJ-DIRECT"}},
+        )
+        assert direct.status_code == 403
+
+    response = await ds.client.post(
+        "/data/-/create",
+        actor={"id": "editor"},
+        json=request_body,
+    )
+    assert response.status_code == 403, response.json()
+    assert response.json()["errors"] == [f"Permission denied: need {denied_action}"]
+    rows = (await db.execute("select id, note from salaries order by id")).rows
+    assert [tuple(r) for r in rows] == [(1, "TOPSECRET-A")]
+    assert await db.table_columns("salaries") == ["id", "note"]
+
+
+@pytest.mark.asyncio
+async def test_create_table_respects_predeclared_table_level_denial():
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                "data": {
+                    "permissions": {
+                        "create-table": {"id": "editor"},
+                        "insert-row": {"id": "editor"},
+                    },
+                    "tables": {
+                        "planned_table": {"permissions": {"insert-row": False}},
+                    },
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database("create_table_predeclared_denial", name="data")
+    await ds.invoke_startup()
+
+    response = await ds.client.post(
+        "/data/-/create",
+        actor={"id": "editor"},
+        json={"table": "planned_table", "rows": [{"id": 1}]},
+    )
+
+    assert response.status_code == 403, response.json()
+    assert response.json()["errors"] == ["Permission denied: need insert-row"]
+    assert not await db.table_exists("planned_table")

tests/test_auth.py

@@ -1,4 +1,5 @@
 import time
+from unittest.mock import AsyncMock
 
 import pytest
 from bs4 import BeautifulSoup as Soup
@@ -237,6 +238,35 @@ def test_auth_create_token(
         assert response3.json["actor"]["id"] == "test"
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("method", ["GET", "POST"])
+@pytest.mark.parametrize(
+    "restrictions",
+    [
+        {},
+        {"a": ["vi"]},
+        {"d": {"db": ["vd"]}},
+        {"r": {"db": {"t1": ["vt"]}}},
+    ],
+    ids=["empty", "instance", "database", "table"],
+)
+async def test_auth_create_token_not_allowed_for_restricted_actors(
+    bare_ds, monkeypatch, method, restrictions
+):
+    create_token = AsyncMock()
+    monkeypatch.setattr(bare_ds, "create_token", create_token)
+
+    response = await bare_ds.client.request(
+        method,
+        "/-/create-token",
+        actor={"id": "test", "_r": restrictions},
+    )
+
+    assert response.status_code == 403
+    assert "Restricted actors cannot create API tokens" in response.text
+    create_token.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_auth_create_token_not_allowed_for_tokens(ds_client):
     ds_tok = ds_client.ds.sign(
@@ -524,3 +554,25 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client):
         )
         is not True
     ), "Root without root_enabled should not automatically get set-column-type"
+
+
+@pytest.mark.parametrize("expire_after", (1, 300, 3600, 30 * 24 * 60 * 60))
+def test_set_actor_cookie_honours_expire_after(expire_after):
+    # GHSA-53fc-rhfg-h7qp issue 4: expire_after is documented as a number of
+    # seconds, but every value was being replaced with 24 hours.
+    from datasette.app import Datasette
+    from datasette.utils.asgi import Response
+
+    ds = Datasette(memory=True)
+    response = Response.text("")
+    before = int(time.time())
+    ds.set_actor_cookie(response, {"id": "test"}, expire_after=expire_after)
+    after = int(time.time())
+
+    (header,) = response._set_cookie_headers
+    assert header.startswith("ds_actor=")
+    value = header[len("ds_actor=") :].split(";", 1)[0]
+    data = ds.unsign(value, "actor")
+    assert data["a"] == {"id": "test"}
+    expires_at = baseconv.base62.decode(data["e"])
+    assert before + expire_after <= expires_at <= after + expire_after

tests/test_background_tasks.py

@@ -0,0 +1,381 @@
+"""
+Tests for datasette.add_background_task() / start_background_tasks() and the
+BackgroundTask / BackgroundTaskSupervisor machinery in
+datasette/background_tasks.py.
+"""
+
+import asyncio
+import contextlib
+import logging
+
+import httpx2
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.plugins import pm
+
+
+async def _drive_lifespan_startup(app):
+    """Send a single lifespan.startup message into app's ASGI lifespan loop
+    and return the list of messages sent back, without ever sending
+    lifespan.shutdown. Copied from tests/test_lifespan.py's helper of the
+    same name - mirrors what a real server does: after startup completes
+    it parks waiting for the next event, and we cancel that wait once
+    we've observed the startup response.
+    """
+    messages_sent = []
+    startup_responded = asyncio.Event()
+    delivered = False
+
+    async def receive():
+        nonlocal delivered
+        if not delivered:
+            delivered = True
+            return {"type": "lifespan.startup"}
+        await asyncio.Event().wait()
+
+    async def send(message):
+        messages_sent.append(message)
+        startup_responded.set()
+
+    task = asyncio.create_task(app({"type": "lifespan"}, receive, send))
+    try:
+        await asyncio.wait_for(startup_responded.wait(), timeout=5)
+    finally:
+        task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await task
+    return messages_sent
+
+
+@pytest.mark.asyncio
+async def test_tasks_registered_in_startup_hook_run_after_lifespan_startup():
+    # Two tasks registered by one plugin's startup hook - order preserved,
+    # both running after lifespan startup completes, and no HTTP request
+    # of any kind is issued anywhere in this test.
+    events = []
+
+    async def task_one(datasette):
+        events.append("task_one")
+        # Wait indefinitely to simulate long-lived background work, keeping
+        # the task "running" for the assertions below until cleanup cancels it.
+        await asyncio.Event().wait()
+
+    async def task_two(datasette):
+        events.append("task_two")
+        await asyncio.Event().wait()
+
+    class TwoTaskPlugin:
+        __name__ = "TwoTaskPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(task_one, name="task-one")
+                datasette.add_background_task(task_two, name="task-two")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(TwoTaskPlugin(), name="two_task_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan_startup(app)
+        assert {"type": "lifespan.startup.complete"} in messages
+
+        handles = ds._background_tasks.tasks()
+        assert [h.name for h in handles] == ["task-one", "task-two"]
+
+        # Let both tasks run their first line of code.
+        await asyncio.sleep(0)
+        assert handles[0].state == "running"
+        assert handles[1].state == "running"
+        assert events == ["task_one", "task_two"]
+    finally:
+        pm.unregister(name="two_task_plugin")
+        await ds._background_tasks.cancel_all(grace=1.0)
+
+
+@pytest.mark.asyncio
+async def test_launch_waits_for_every_startup_hook_before_running_any_task():
+    # PluginA registers a task from its startup hook; PluginB does the
+    # same from ITS startup hook, which runs after PluginA's (forced with
+    # tryfirst=True on A). Even though A's registration happens first,
+    # A's task body must not actually execute until every startup hook -
+    # including B's - has finished, since launch only happens after
+    # invoke_startup() completes. This is the ordering guarantee that
+    # dissolves datasette-cron's tryfirst=True launch hack.
+    hook_call_order = []
+    seen_names_when_a_ran = {}
+
+    async def task_a(datasette):
+        seen_names_when_a_ran["names"] = [
+            h.name for h in datasette._background_tasks.tasks()
+        ]
+
+    async def task_b(datasette):
+        pass
+
+    class PluginA:
+        __name__ = "PluginA"
+
+        @hookimpl(tryfirst=True)
+        def startup(self, datasette):
+            async def inner():
+                hook_call_order.append("A")
+                datasette.add_background_task(task_a, name="task-a")
+
+            return inner
+
+    class PluginB:
+        __name__ = "PluginB"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                hook_call_order.append("B")
+                datasette.add_background_task(task_b, name="task-b")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(PluginA(), name="plugin_a")
+    pm.register(PluginB(), name="plugin_b")
+    try:
+        await ds.start_background_tasks()
+        # Confirm A's startup hook really did run (and register task-a)
+        # strictly before B's startup hook ran.
+        assert hook_call_order == ["A", "B"]
+
+        handles = ds._background_tasks.tasks()
+        await asyncio.wait_for(asyncio.gather(*[h.task for h in handles]), timeout=5)
+        # Yet by the time task-a's own body executed (after launch, which
+        # only happens once every startup hook - including B's - has
+        # finished), task-b was already registered.
+        assert "task-b" in seen_names_when_a_ran["names"]
+    finally:
+        pm.unregister(name="plugin_a")
+        pm.unregister(name="plugin_b")
+
+
+@pytest.mark.asyncio
+async def test_concurrent_first_requests_launch_background_tasks_exactly_once():
+    launch_count = {"n": 0}
+
+    async def counting_task(datasette):
+        launch_count["n"] += 1
+
+    class CountingTaskPlugin:
+        __name__ = "CountingTaskPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(counting_task, name="counting-task")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(CountingTaskPlugin(), name="counting_task_plugin")
+    try:
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            responses = await asyncio.gather(
+                *[client.get("/-/versions.json") for _ in range(10)]
+            )
+        assert all(response.status_code == 200 for response in responses)
+
+        handles = ds._background_tasks.tasks()
+        assert len(handles) == 1
+        await asyncio.wait_for(handles[0].task, timeout=5)
+        assert launch_count["n"] == 1
+    finally:
+        pm.unregister(name="counting_task_plugin")
+
+
+@pytest.mark.asyncio
+async def test_post_launch_registration_starts_immediately_and_cancel_works():
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()  # nothing registered yet, but launched
+
+    started = asyncio.Event()
+
+    async def long_running(datasette):
+        started.set()
+        await asyncio.Event().wait()
+
+    handle = ds.add_background_task(long_running, name="dynamic-task")
+    # Registered after launch: starts immediately rather than sitting in
+    # "registered" limbo.
+    assert handle.state == "running"
+    assert handle.task is not None
+
+    await asyncio.wait_for(started.wait(), timeout=5)
+    assert handle.state == "running"
+
+    handle.cancel()
+    with pytest.raises(asyncio.CancelledError):
+        await handle.task
+    await asyncio.sleep(0)
+    assert handle.state == "cancelled"
+
+
+@pytest.mark.asyncio
+async def test_pre_launch_registration_starts_as_registered():
+    ds = Datasette(memory=True)
+
+    async def task(datasette):
+        pass
+
+    handle = ds.add_background_task(task, name="buffered-task")
+    assert handle.state == "registered"
+    assert handle.task is None
+
+    handle.cancel()  # not yet launched: deregisters instead of cancelling
+    assert handle not in ds._background_tasks.tasks()
+
+
+@pytest.mark.asyncio
+async def test_crashing_task_logs_traceback_and_state_is_crashed(caplog):
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()
+
+    survivor_ran = asyncio.Event()
+
+    async def crashing_task(datasette):
+        raise RuntimeError("kaboom")
+
+    async def survivor(datasette):
+        survivor_ran.set()
+
+    with caplog.at_level(logging.ERROR, logger="datasette.background_tasks"):
+        crash_handle = ds.add_background_task(crashing_task, name="crashing_task")
+        survivor_handle = ds.add_background_task(survivor, name="survivor")
+        await asyncio.wait_for(
+            asyncio.gather(
+                crash_handle.task, survivor_handle.task, return_exceptions=True
+            ),
+            timeout=5,
+        )
+
+    assert crash_handle.state == "crashed"
+    assert isinstance(crash_handle.exception, RuntimeError)
+    assert str(crash_handle.exception) == "kaboom"
+
+    # The crash must not affect any other task.
+    assert survivor_ran.is_set()
+    assert survivor_handle.state == "completed"
+
+    assert "crashing_task" in caplog.text
+    assert "kaboom" in caplog.text
+    assert "Traceback" in caplog.text
+    assert "RuntimeError" in caplog.text
+
+
+def test_name_collisions_get_suffixed_and_explicit_names_are_respected():
+    ds = Datasette(memory=True)
+
+    async def noop(datasette):
+        pass
+
+    async def another_noop(datasette):
+        pass
+
+    h1 = ds.add_background_task(noop, name="dup")
+    h2 = ds.add_background_task(another_noop, name="dup")
+    h3 = ds.add_background_task(noop, name="dup")
+    assert [h1.name, h2.name, h3.name] == ["dup", "dup-2", "dup-3"]
+
+    h_explicit = ds.add_background_task(noop, name="explicit-name")
+    assert h_explicit.name == "explicit-name"
+
+    h_default = ds.add_background_task(noop)
+    assert h_default.name == noop.__qualname__
+
+
+@pytest.mark.asyncio
+async def test_start_background_tasks_on_bare_datasette():
+    # The headless-CLI path (datasette-rss's `fetch --due` shape): no
+    # server, no lifespan, no first HTTP request - just an explicit call.
+    ran = asyncio.Event()
+
+    async def task(datasette):
+        ran.set()
+
+    ds = Datasette([])
+    assert ds._startup_invoked is False
+
+    handle = ds.add_background_task(task, name="headless-task")
+    assert handle.state == "registered"
+
+    await ds.start_background_tasks()
+
+    assert ds._startup_invoked is True
+    await asyncio.wait_for(ran.wait(), timeout=5)
+    await asyncio.wait_for(handle.task, timeout=5)
+    # handle.task being done only guarantees the coroutine has returned,
+    # not that our done-callback (which updates handle.state) has run yet -
+    # asyncio schedules done-callbacks via call_soon, and awaiting an
+    # already-done future/task returns immediately without giving the loop
+    # a chance to drain its ready queue. Yield once to let it run.
+    await asyncio.sleep(0)
+    assert handle.state == "completed"
+
+
+@pytest.mark.asyncio
+async def test_cancel_all_cancels_running_tasks_and_leaves_completed_alone():
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()
+
+    async def forever(datasette):
+        await asyncio.Event().wait()
+
+    async def quick(datasette):
+        return "done"
+
+    forever_handle = ds.add_background_task(forever, name="forever")
+    quick_handle = ds.add_background_task(quick, name="quick")
+    await asyncio.wait_for(quick_handle.task, timeout=5)
+    assert quick_handle.state == "completed"
+
+    await ds._background_tasks.cancel_all(grace=1.0)
+
+    assert forever_handle.state == "cancelled"
+    assert quick_handle.state == "completed"
+
+
+@pytest.mark.asyncio
+async def test_cancel_all_logs_stragglers_that_outlive_the_grace_period(caplog):
+    ds = Datasette(memory=True)
+    await ds.start_background_tasks()
+
+    async def stubborn(datasette):
+        with contextlib.suppress(asyncio.CancelledError):
+            await asyncio.sleep(10)
+        # Swallowing CancelledError above and returning normally simulates
+        # a task that ignores cancellation for longer than the grace period.
+        await asyncio.sleep(10)
+
+    handle = ds.add_background_task(stubborn, name="stubborn-task")
+    # Let the task actually start running and reach its first sleep (inside
+    # the CancelledError-suppressing block) before cancelling it - a task
+    # cancelled before it has ever run its first step never enters that
+    # block at all (the throw happens before the coroutine body starts),
+    # so it would finish cancelling immediately instead of behaving like a
+    # straggler.
+    await asyncio.sleep(0)
+
+    with caplog.at_level(logging.WARNING, logger="datasette.background_tasks"):
+        await ds._background_tasks.cancel_all(grace=0.1)
+
+    assert "stubborn-task" in caplog.text
+
+    # Clean up: actually cancel it now that the test has made its
+    # assertion, so it doesn't leak past the end of the test.
+    handle.task.cancel()
+    with contextlib.suppress(asyncio.CancelledError):
+        await handle.task

tests/test_cli_serve_get.py

@@ -52,6 +52,56 @@ def test_serve_with_get(tmp_path_factory):
     pm.unregister(to_unregister)
 
 
+def test_serve_with_get_does_not_launch_background_tasks(tmp_path_factory):
+    # --get must never launch background tasks, even though its TestClient
+    # request
+    # flows through the full ASGI stack (including the AsgiRunOnFirstRequest
+    # fallback that would otherwise launch them). The plugin's startup hook
+    # itself still runs (registration happens) - only the launch is
+    # suppressed, so the sentinel file the background task would write must
+    # never appear.
+    plugins_dir = tmp_path_factory.mktemp("plugins_for_get_background_tasks")
+    sentinel = plugins_dir / "sentinel.txt"
+    (plugins_dir / "bg_task_for_get.py").write_text(
+        textwrap.dedent(
+            f"""
+        from datasette import hookimpl
+
+        @hookimpl
+        def startup(datasette):
+            async def inner():
+                async def task(datasette):
+                    with open("{sentinel!s}", "w") as fp:
+                        fp.write("ran")
+
+                datasette.add_background_task(task, name="get-sentinel-task")
+
+            return inner
+    """,
+        ),
+        "utf-8",
+    )
+    runner = CliRunner()
+    result = runner.invoke(
+        cli,
+        [
+            "serve",
+            "--memory",
+            "--plugins-dir",
+            str(plugins_dir),
+            "--get",
+            "/_memory/-/query.json?sql=select+1",
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    assert not sentinel.exists()
+
+    to_unregister = next(
+        p for p in pm.get_plugins() if p.__name__ == "bg_task_for_get.py"
+    )
+    pm.unregister(to_unregister)
+
+
 def test_serve_with_get_headers():
     runner = CliRunner()
     result = runner.invoke(

tests/test_cli_serve_server.py

@@ -1,12 +1,15 @@
+import signal
 import socket
+import subprocess
+import time
 
-import httpx
+import httpx2
 import pytest
 
 
 @pytest.mark.serial
 def test_serve_localhost_http(ds_localhost_http_server):
-    response = httpx.get("http://localhost:8041/_memory.json")
+    response = httpx2.get("http://localhost:8041/_memory.json")
     assert {
         "database": "_memory",
         "path": "/_memory",
@@ -20,11 +23,205 @@ def test_serve_localhost_http(ds_localhost_http_server):
 )
 def test_serve_unix_domain_socket(ds_unix_domain_socket_server):
     _, uds = ds_unix_domain_socket_server
-    transport = httpx.HTTPTransport(uds=uds)
+    transport = httpx2.HTTPTransport(uds=uds)
-    client = httpx.Client(transport=transport)
+    client = httpx2.Client(transport=transport)
     response = client.get("http://localhost/_memory.json")
     assert {
         "database": "_memory",
         "path": "/_memory",
         "tables": [],
     }.items() <= response.json().items()
+
+
+# Shaped after datasette-litestream's startup hook, which schedules a
+# background task with asyncio.get_running_loop().create_task(...):
+# https://github.com/datasette/datasette-litestream
+MARKER_TASK_PLUGIN = """
+import asyncio
+from datasette import hookimpl
+from datasette.utils.asgi import Response
+
+
+@hookimpl
+def startup(datasette):
+    datasette._startup_calls = getattr(datasette, "_startup_calls", 0) + 1
+
+    async def _mark():
+        # Must await before setting the flag: a task with no internal
+        # await point could finish on the throwaway loop before it
+        # closed, masking the regression this test guards against.
+        await asyncio.sleep(0.2)
+        datasette._marker_task_ran = True
+
+    asyncio.get_running_loop().create_task(_mark())
+
+
+@hookimpl
+def register_routes():
+    async def marker_status(datasette):
+        return Response.json(
+            {
+                "marker_task_ran": getattr(datasette, "_marker_task_ran", False),
+                "startup_calls": getattr(datasette, "_startup_calls", 0),
+            }
+        )
+
+    return [(r"^/-/marker-task-ran$", marker_status)]
+"""
+
+
+STARTUP_ERROR_PLUGIN = """
+from datasette import hookimpl
+from datasette.utils import StartupError
+
+
+@hookimpl
+def startup(datasette):
+    raise StartupError("boom from plugin")
+"""
+
+
+@pytest.mark.serial
+def test_startup_hook_background_task_runs_on_serving_loop(serve_with_plugins):
+    """
+    Litestream-shaped regression test: a startup hook that does
+    asyncio.get_running_loop().create_task(...) must have that task
+    actually execute before/while the server is handling requests. This
+    only holds if invoke_startup() and uvicorn.Server.serve() share one
+    event loop. This test fails against unmodified main, where
+    invoke_startup() runs on a throwaway loop that is closed before
+    uvicorn opens its own loop to serve.
+    """
+    _, port = serve_with_plugins({"marker_task_plugin": MARKER_TASK_PLUGIN})
+    # The fixture has already waited for the server to answer requests. The
+    # marker task deliberately awaits before setting its flag, so poll for a
+    # moment rather than assuming it landed before the first request arrived.
+    deadline = time.time() + 3.0
+    payload = {}
+    while time.time() < deadline:
+        payload = httpx2.get(
+            f"http://127.0.0.1:{port}/-/marker-task-ran", timeout=1.0
+        ).json()
+        if payload["marker_task_ran"]:
+            break
+        time.sleep(0.05)
+    assert payload.get("marker_task_ran"), (
+        "The startup hook's asyncio.create_task(...) never ran - "
+        "invoke_startup() and the server are not sharing an event loop"
+    )
+    # Polling above means this test would also pass if the startup hook were
+    # re-run on the serving loop by the first-request fallback - which would
+    # hide exactly the bug being tested. invoke_startup() is idempotent today
+    # so that cannot happen; assert it explicitly so that if the idempotency
+    # guard is ever removed this test fails loudly instead of silently
+    # becoming a no-op.
+    assert payload["startup_calls"] == 1, (
+        "startup hook ran {} times - the marker may have been set by a "
+        "re-run on the serving loop rather than by the original task".format(
+            payload["startup_calls"]
+        )
+    )
+
+
+@pytest.mark.serial
+def test_startup_error_fails_fast_before_port_binds(serve_with_plugins):
+    """
+    A "startup" plugin hook that raises StartupError must fail fast: print
+    the message, exit non-zero, and never accept a connection on the port -
+    the failure must happen before uvicorn.Server binds the socket.
+    """
+    proc, port = serve_with_plugins(
+        {"startup_error_plugin": STARTUP_ERROR_PLUGIN}, wait_for_startup=False
+    )
+    stdout, _ = proc.communicate(timeout=15)
+    output = stdout.decode("utf-8")
+    assert proc.returncode not in (0, None), output
+    assert "boom from plugin" in output, output
+
+    # Nothing is listening on the port now the process has exited. This
+    # confirms the socket was not left bound; on its own it cannot prove the
+    # failure preceded the bind, since a port nothing ever touched also
+    # refuses connections.
+    with (
+        pytest.raises(OSError),
+        socket.create_connection(("127.0.0.1", port), timeout=0.2),
+    ):
+        pass
+
+
+# Verify that SIGTERM and SIGINT sent to `datasette serve` trigger uvicorn's
+# lifespan.shutdown event and run the plugin shutdown hooks. The plugin below
+# writes a sentinel file from its shutdown hook so the tests can check that
+# cleanup ran after the server subprocess exits.
+SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE = """
+import pathlib
+from datasette import hookimpl
+
+SENTINEL_PATH = {sentinel_path!r}
+
+
+@hookimpl
+def shutdown(datasette):
+    pathlib.Path(SENTINEL_PATH).write_text("shutdown ran", "utf-8")
+"""
+
+
+def _start_serve_with_shutdown_sentinel(serve_with_plugins, tmp_path):
+    sentinel_path = tmp_path / "shutdown-sentinel.txt"
+    proc, _ = serve_with_plugins(
+        {
+            "shutdown_sentinel_plugin": SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE.format(
+                sentinel_path=str(sentinel_path)
+            )
+        }
+    )
+    return proc, sentinel_path
+
+
+@pytest.mark.serial
+def test_sigterm_runs_shutdown_hooks(serve_with_plugins, tmp_path):
+    ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel(
+        serve_with_plugins, tmp_path
+    )
+    assert not sentinel_path.exists()
+    ds_proc.send_signal(signal.SIGTERM)
+    try:
+        ds_proc.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        ds_proc.kill()
+        ds_proc.wait()
+        raise AssertionError(
+            "datasette serve did not exit within 10s of SIGTERM\n"
+            + ds_proc.stdout.read().decode("utf-8")
+        )
+    output = ds_proc.stdout.read().decode("utf-8")
+    assert sentinel_path.exists(), (
+        "shutdown hook never wrote its sentinel file after SIGTERM\n" + output
+    )
+    assert sentinel_path.read_text("utf-8") == "shutdown ran"
+
+
+@pytest.mark.serial
+@pytest.mark.skipif(
+    not hasattr(signal, "SIGINT"), reason="Requires signal.SIGINT support"
+)
+def test_sigint_runs_shutdown_hooks(serve_with_plugins, tmp_path):
+    ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel(
+        serve_with_plugins, tmp_path
+    )
+    assert not sentinel_path.exists()
+    ds_proc.send_signal(signal.SIGINT)
+    try:
+        ds_proc.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        ds_proc.kill()
+        ds_proc.wait()
+        raise AssertionError(
+            "datasette serve did not exit within 10s of SIGINT\n"
+            + ds_proc.stdout.read().decode("utf-8")
+        )
+    output = ds_proc.stdout.read().decode("utf-8")
+    assert sentinel_path.exists(), (
+        "shutdown hook never wrote its sentinel file after SIGINT\n" + output
+    )
+    assert sentinel_path.read_text("utf-8") == "shutdown ran"

tests/test_fts_permissions.py

@@ -0,0 +1,412 @@
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.permissions import Action, PermissionSQL, _permission_check_cache
+from datasette.resources import DatabaseResource, TableResource
+from datasette.utils.sqlite import sqlite3, sqlite_derived_table_dependencies
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ["fts4", "fts5"])
+@pytest.mark.parametrize("actor", [None, {"id": "root"}], ids=["anonymous", "root"])
+async def test_derived_permissions_allow_one_hop_but_deny_nested_sources(
+    fts_module, actor
+):
+    class InspectPlugin:
+        @hookimpl
+        def register_actions(self):
+            return [
+                Action(
+                    name="inspect-derived",
+                    description="Inspect a table",
+                    resource_class=TableResource,
+                    also_requires="view-table",
+                )
+            ]
+
+        @hookimpl
+        def permission_resources_sql(self, action):
+            if action == "inspect-derived":
+                return PermissionSQL(
+                    sql="SELECT NULL AS parent, NULL AS child, 1 AS allow, 'inspect allowed' AS reason"
+                )
+
+    ds = Datasette(memory=True)
+    ds.pm.register(InspectPlugin(), name="inspect-derived-test")
+    db = ds.add_memory_database(
+        f"derived_one_hop_{fts_module}_{actor is not None}", name="data"
+    )
+    await db.execute_write("create table Documents (body text)")
+    await db.execute_write(
+        f"create virtual table Search using {fts_module}(body, content='Documents')"
+    )
+    await db.execute_write(
+        f"create virtual table Nested using {fts_module}(body, content='sEaRcH')"
+    )
+    await ds.invoke_startup()
+    token = _permission_check_cache.set({})
+    try:
+        # Both direct permissions are allowed, but a derived source makes its
+        # dependent unavailable even to an actor who can view the whole chain.
+        # Check and cache Search first so its cached grant cannot grant Nested.
+        for table, expected in (
+            ("Documents", True),
+            ("Search", True),
+            ("Nested", False),
+            ("Search_docsize", False),
+        ):
+            for spelling in (table, table.upper(), table.lower()):
+                assert await ds.allowed_many(
+                    actions=["view-table", "inspect-derived"],
+                    resource=TableResource("data", spelling),
+                    actor=actor,
+                ) == {"view-table": expected, "inspect-derived": expected}
+
+        page = await ds.allowed_resources(
+            "view-table", actor, parent="data", include_is_private=True, limit=1000
+        )
+        allowed = {resource.child for resource in page.resources}
+        assert {"Documents", "Search"}.issubset(allowed)
+        assert "Nested" not in allowed
+        assert "Search_docsize" not in allowed
+    finally:
+        _permission_check_cache.reset(token)
+        ds.pm.unregister(name="inspect-derived-test")
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("listing", [False, True], ids=["individual", "listing"])
+async def test_derived_permission_discovery_error_is_retried(monkeypatch, listing):
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(f"derived_discovery_error_{listing}", name="data")
+    await db.execute_write("create table documents (id integer primary key)")
+    await ds.invoke_startup()
+
+    class UnavailableSchema:
+        def execute(self, sql):
+            raise sqlite3.DatabaseError("schema temporarily unavailable")
+
+    async def check():
+        if listing:
+            return await ds.allowed_resources("view-table", parent="data")
+        return await ds.allowed(
+            action="view-table", resource=TableResource("data", "documents")
+        )
+
+    token = _permission_check_cache.set({})
+    try:
+        with monkeypatch.context() as patch:
+            patch.setattr(
+                "datasette.database.sqlite_derived_table_dependencies",
+                lambda conn: sqlite_derived_table_dependencies(UnavailableSchema()),
+            )
+            with pytest.raises(sqlite3.DatabaseError, match="schema temporarily"):
+                await check()
+
+        # Failed discovery must not cache an empty map or a permission grant.
+        assert db._cached_derived_table_dependencies is None
+        assert not _permission_check_cache.get()
+        result = await check()
+        if listing:
+            assert [resource.child for resource in result.resources] == ["documents"]
+        else:
+            assert result is True
+        assert db._cached_derived_table_dependencies is not None
+    finally:
+        _permission_check_cache.reset(token)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ("fts4", "fts5"))
+async def test_external_content_fts_inherits_content_table_view_permission(fts_module):
+    actor = {"id": "reader"}
+    secret_marker = "ISSUE_17_EXTERNAL_CONTENT_FTS_SECRET"
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": {"id": "reader"},
+                "view-database": {"id": "reader"},
+                "view-table": {"id": "reader"},
+                "execute-sql": {"id": "nobody"},
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": False}},
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database(f"issue_17_{fts_module}_permissions", name="data")
+    await db.execute_write("create table secret (id integer primary key, body text)")
+    await db.execute_write(
+        "insert into secret (body) values (?)",
+        [secret_marker],
+    )
+    fts_options = "body, content='secret'"
+    if fts_module == "fts5":
+        fts_options += ", content_rowid='id'"
+    await db.execute_write(
+        f"create virtual table secret_fts using {fts_module}({fts_options})"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        assert "secret_fts" in await db.hidden_table_names()
+        assert (
+            await ds.allowed(
+                action="execute-sql",
+                resource=DatabaseResource("data"),
+                actor=actor,
+            )
+            is False
+        )
+
+        direct = await ds.client.get("/data/secret.json", actor=actor)
+        assert direct.status_code == 403
+
+        companion = await ds.client.get(
+            "/data/secret_fts.json?_shape=array",
+            actor=actor,
+        )
+        assert companion.status_code in (403, 404), (
+            "An automatically hidden external-content FTS table must inherit "
+            "the content table's view denial or be unavailable: "
+            f"{companion.text}"
+        )
+        assert secret_marker not in companion.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("fts_module", ("fts4", "fts5"))
+@pytest.mark.parametrize("contentless", (False, True), ids=("internal", "contentless"))
+async def test_fts_shadow_tables_inherit_logical_table_view_permission(
+    fts_module, contentless
+):
+    table_config = {
+        "secret_fts": {"permissions": {"view-table": False}},
+        # An explicit allow on one implementation table must not override
+        # the logical FTS table's denial.
+        "secret_fts_docsize": {"permissions": {"view-table": True}},
+    }
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+                "view-table": True,
+                "execute-sql": False,
+            },
+            "databases": {"data": {"tables": table_config}},
+        },
+    )
+    db = ds.add_memory_database(
+        f"issue_17_{fts_module}_{'contentless' if contentless else 'internal'}",
+        name="data",
+    )
+    options = "body, content=''" if contentless else "body"
+    await db.execute_write(
+        f"create virtual table secret_fts using {fts_module}({options})"
+    )
+    await db.execute_write(
+        "insert into secret_fts(rowid, body) values (1, 'ISSUE_17_SHADOW_SECRET')"
+    )
+    await ds.invoke_startup()
+
+    try:
+        dependencies = await db.derived_table_dependencies()
+        shadow_tables = sorted(
+            table for table, source in dependencies.items() if source == "secret_fts"
+        )
+        assert shadow_tables
+        assert "secret_fts_docsize" in shadow_tables
+
+        for shadow_table in shadow_tables:
+            assert (
+                await ds.allowed(
+                    action="view-table",
+                    resource=TableResource("data", shadow_table),
+                )
+                is False
+            )
+            response = await ds.client.get(f"/data/{shadow_table}.json?_shape=array")
+            assert response.status_code == 403
+            assert "ISSUE_17_SHADOW_SECRET" not in response.text
+
+        allowed = await ds.allowed_resources("view-table", parent="data", limit=1000)
+        allowed_names = {resource.child for resource in allowed.resources}
+        assert not set(shadow_tables).intersection(allowed_names)
+
+        database_json = await ds.client.get("/data.json")
+        assert database_json.status_code == 200
+        for shadow_table in shadow_tables:
+            assert shadow_table not in database_json.text
+
+        schema_json = await ds.client.get("/data/-/schema.json")
+        assert schema_json.status_code == 200
+        for shadow_table in shadow_tables:
+            assert shadow_table not in schema_json.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "content_allowed,companion_allowed,expected",
+    (
+        (False, True, False),
+        (True, False, False),
+        (True, True, True),
+    ),
+)
+async def test_external_content_and_companion_permissions_are_both_required(
+    content_allowed, companion_allowed, expected
+):
+    ds = Datasette(
+        memory=True,
+        default_deny=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": content_allowed}},
+                        "secret_fts": {
+                            "permissions": {"view-table": companion_allowed}
+                        },
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database(
+        f"issue_17_explicit_{int(content_allowed)}_{int(companion_allowed)}",
+        name="data",
+    )
+    await db.execute_write("create table secret(id integer primary key, body text)")
+    await db.execute_write("insert into secret(body) values ('ISSUE_17_MATRIX_SECRET')")
+    await db.execute_write(
+        "create virtual table secret_fts using fts5("
+        "body, content='secret', content_rowid='id')"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        assert (
+            await ds.allowed(
+                action="view-table",
+                resource=TableResource("data", "secret_fts"),
+            )
+            is expected
+        )
+        response = await ds.client.get("/data/secret_fts.json?_shape=array")
+        assert response.status_code == (200 if expected else 403)
+        if not expected:
+            assert "ISSUE_17_MATRIX_SECRET" not in response.text
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_derived_tables_propagate_private_flag_and_route_permissions():
+    actor = {"id": "reader"}
+    ds = Datasette(
+        memory=True,
+        config={
+            "permissions": {
+                "view-instance": True,
+                "view-database": True,
+                "view-table": True,
+            },
+            "databases": {
+                "data": {
+                    "tables": {
+                        "secret": {"permissions": {"view-table": {"id": "reader"}}}
+                    }
+                }
+            },
+        },
+    )
+    db = ds.add_memory_database("issue_17_private_flag", name="data")
+    await db.execute_write("create table secret(id integer primary key, body text)")
+    await db.execute_write("insert into secret(body) values ('PRIVATE')")
+    await db.execute_write(
+        "create virtual table secret_fts using fts5("
+        "body, content='secret', content_rowid='id')"
+    )
+    await db.execute_write("insert into secret_fts(secret_fts) values ('rebuild')")
+    await ds.invoke_startup()
+
+    try:
+        actor_page = await ds.allowed_resources(
+            "view-table", actor, parent="data", include_is_private=True, limit=1000
+        )
+        actor_resources = {
+            resource.child: resource for resource in actor_page.resources
+        }
+        derived_names = set(await db.derived_table_dependencies())
+        assert "secret_fts" in actor_resources
+        assert actor_resources["secret_fts"].private
+        # Shadow tables depend on the already-derived external-content FTS
+        # table, so they remain unavailable even to the permitted reader.
+        assert not (derived_names - {"secret_fts"}).intersection(actor_resources)
+
+        anonymous_page = await ds.allowed_resources(
+            "view-table", parent="data", limit=1000
+        )
+        anonymous_names = {resource.child for resource in anonymous_page.resources}
+        assert not derived_names.intersection(anonymous_names)
+
+        for path in (
+            "/data/secret_fts.json?_facet=body",
+            "/data/secret_fts.csv",
+            "/data/secret_fts/-/autocomplete?q=PRIVATE",
+            "/data/secret_fts/-/schema.json",
+        ):
+            denied = await ds.client.get(path)
+            assert denied.status_code == 403
+            allowed = await ds.client.get(path, actor=actor)
+            assert allowed.status_code == 200
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_cyclic_derived_table_dependencies_fail_closed():
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database("issue_17_cycle", name="data")
+    await db.execute_write(
+        "create virtual table first_fts using fts5(body, content='second_fts')"
+    )
+    await db.execute_write(
+        "create virtual table second_fts using fts5(body, content='first_fts')"
+    )
+    await ds.invoke_startup()
+
+    try:
+        for table in ("first_fts", "second_fts"):
+            assert (
+                await ds.allowed(
+                    action="view-table", resource=TableResource("data", table)
+                )
+                is False
+            )
+
+        page = await ds.allowed_resources("view-table", parent="data", limit=1000)
+        allowed_names = {resource.child for resource in page.resources}
+        assert "first_fts" not in allowed_names
+        assert "second_fts" not in allowed_names
+    finally:
+        ds.close()

tests/test_html.py

@@ -36,8 +36,10 @@ def test_homepage(app_client_two_attached_databases):
     h2 = soup.select("h2")[0]
     assert "extra database" == h2.text.strip()
     counts_p, links_p = h2.find_all_next("p")[:2]
+    # Shadow tables of the external-content index are denied, so they do not
+    # contribute to the table or row totals.
     assert (
-        "2 rows in 1 table, 5 rows in 4 hidden tables, 1 view" == counts_p.text.strip()
+        "2 rows in 1 table, 2 rows in 1 hidden table, 1 view" == counts_p.text.strip()
     )
     # We should only show visible, not hidden tables here:
     table_links = [

tests/test_internals_database.py

@@ -15,11 +15,16 @@ from datasette.database import (
     DatasetteClosedError,
     ExecuteWriteResult,
     MultipleValues,
+    QueryInterrupted,
     Results,
     _deliver_write_result,
 )
 from datasette.utils import Column
-from datasette.utils.sqlite import sqlite3, supports_returning
+from datasette.utils.sqlite import (
+    sqlite3,
+    sqlite_derived_table_dependencies,
+    supports_returning,
+)
 
 requires_sqlite_returning = pytest.mark.skipif(
     not supports_returning(), reason="SQLite does not support RETURNING"
@@ -38,6 +43,31 @@ async def test_execute(db):
     assert 15 == len(results)
 
 
+@pytest.mark.asyncio
+async def test_derived_dependency_cache_survives_failed_refresh(monkeypatch):
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.derived_table_dependencies()
+    previous_cache = db._cached_derived_table_dependencies
+    await db.execute_write("create table dependency_cache_refresh (id integer)")
+
+    class UnavailableSchema:
+        def execute(self, sql):
+            raise sqlite3.DatabaseError("schema temporarily unavailable")
+
+    with monkeypatch.context() as patch:
+        patch.setattr(
+            "datasette.database.sqlite_derived_table_dependencies",
+            lambda conn: sqlite_derived_table_dependencies(UnavailableSchema()),
+        )
+        with pytest.raises(sqlite3.DatabaseError, match="schema temporarily"):
+            await db.derived_table_dependencies()
+    assert db._cached_derived_table_dependencies == previous_cache
+
+    await db.derived_table_dependencies()
+    assert db._cached_derived_table_dependencies[0] != previous_cache[0]
+
+
 @pytest.mark.asyncio
 async def test_results_first(db):
     assert None is (await db.execute("select * from facetable where pk > 100")).first()
@@ -478,6 +508,31 @@ async def test_view_names(db):
     ]
 
 
+@pytest.mark.asyncio
+async def test_execute_write_custom_time_limit():
+    ds = Datasette(settings={"sql_time_limit_ms": 1})
+    db = ds.add_memory_database(uuid.uuid4().hex, name="write_limits")
+    await ds.invoke_startup()
+    # Bounded work from PR #51; even without a limit this finishes on its own.
+    sql = (
+        "with recursive c(x) as "
+        "(select 1 union all select x+1 from c where x < 800000) "
+        "select x from c where x < 0"
+    )
+    try:
+        await db.execute_write("create table items(value integer)")
+        with pytest.raises(QueryInterrupted):
+            await db.execute(sql)
+        # Writes take their own explicit limit, independent of the read setting.
+        with pytest.raises(QueryInterrupted):
+            await db.execute_write(f"insert into items(value) {sql}", time_limit_ms=1)
+        # Interruption must leave the connection available for subsequent writes.
+        await db.execute_write("insert into items(value) values (1)")
+        assert (await db.execute("select value from items")).single_value() == 1
+    finally:
+        ds.close()
+
+
 @pytest.mark.asyncio
 async def test_execute_write_block_true(db):
     result = await db.execute_write(
@@ -705,6 +760,33 @@ async def test_execute_write_fn_block_false(db):
     assert isinstance(task_id, uuid.UUID)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("disable_threads", (False, True))
+async def test_execute_write_fn_block_false_returns_uuid(tmp_path, disable_threads):
+    # block=False is documented to return "a UUID representing the queued task".
+    # With num_sql_threads=0 there is no write thread, so the non-threaded branch
+    # has to satisfy the same contract as the threaded one.
+    settings = {"num_sql_threads": 0} if disable_threads else {}
+    ds = Datasette([], memory=True, settings=settings)
+    await ds.invoke_startup()
+    db = ds.add_memory_database("test_block_false")
+    await db.execute_write(
+        "create table if not exists t (id integer primary key, v text)"
+    )
+
+    def write_fn(conn):
+        conn.execute("insert into t (v) values ('a')")
+        # Returns None, like most write functions.
+
+    task_id = await db.execute_write_fn(write_fn, block=False)
+
+    assert isinstance(task_id, uuid.UUID)
+    # Distinct per call, so a caller can tell two queued tasks apart.
+    second = await db.execute_write_fn(write_fn, block=False)
+    assert isinstance(second, uuid.UUID)
+    assert second != task_id
+
+
 @pytest.mark.asyncio
 async def test_execute_write_fn_block_true(db):
     def write_fn(conn):

tests/test_internals_datasette_client.py

@@ -1,4 +1,4 @@
-import httpx
+import httpx2
 import pytest
 import pytest_asyncio
 
@@ -43,7 +43,7 @@ async def datasette_with_permissions():
 async def test_client_methods(datasette, method, path, expected_status):
     client_method = getattr(datasette.client, method)
     response = await client_method(path)
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
     assert response.status_code == expected_status
     # Try that again using datasette.client.request
     response2 = await datasette.client.request(method, path)
@@ -63,7 +63,7 @@ async def test_client_post(datasette, prefix):
                 "message": "A message",
             },
         )
-        assert isinstance(response, httpx.Response)
+        assert isinstance(response, httpx2.Response)
         assert response.status_code == 302
         assert "ds_messages" in response.cookies
     finally:
@@ -135,7 +135,7 @@ async def test_skip_permission_checks_all_methods(datasette_with_permissions, me
     response = await client_method("/test_db.json", skip_permission_checks=True)
     # We don't check status code since some methods might not be allowed,
     # but we verify the request doesn't fail due to permissions
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
 
 
 @pytest.mark.asyncio
@@ -340,7 +340,7 @@ async def test_actor_parameter_all_http_methods(datasette, method):
     client_method = getattr(datasette.client, method)
     # Just verify no TypeError about unexpected 'actor' kwarg
     response = await client_method("/", actor={"id": "root"})
-    assert isinstance(response, httpx.Response)
+    assert isinstance(response, httpx2.Response)
 
 
 @pytest.mark.asyncio

tests/test_lifespan.py

@@ -0,0 +1,344 @@
+"""
+Tests for wiring Datasette startup (setup_db table counts + invoke_startup)
+into the ASGI lifespan protocol.
+
+These exercise Datasette._startup_sequence() via three different callers:
+- AsgiLifespan, by hand-driving lifespan.startup messages (no HTTP request)
+- AsgiRunOnFirstRequest, the fallback for hosts that never send lifespan
+  events (this is what DatasetteClient / plain httpx2.ASGITransport uses)
+- Both at once, to prove startup hooks run at most once
+"""
+
+import asyncio
+import contextlib
+import sqlite3
+
+import httpx2
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.database import Database
+from datasette.plugins import pm
+
+
+async def _drive_lifespan_startup(app):
+    """Send a single lifespan.startup message into app's ASGI lifespan loop
+    and return the list of messages sent back - without ever sending
+    lifespan.shutdown. Mirrors what a real server does: after startup
+    completes it parks waiting for the next event. We cancel that wait
+    once we've observed the startup response, rather than closing the
+    Datasette instance down with a shutdown message.
+    """
+    messages_sent = []
+    startup_responded = asyncio.Event()
+    delivered = False
+
+    async def receive():
+        nonlocal delivered
+        if not delivered:
+            delivered = True
+            return {"type": "lifespan.startup"}
+        # No further messages: block until the task is cancelled below,
+        # same as a real server parked waiting for lifespan.shutdown.
+        await asyncio.Event().wait()
+
+    async def send(message):
+        messages_sent.append(message)
+        startup_responded.set()
+
+    task = asyncio.create_task(app({"type": "lifespan"}, receive, send))
+    try:
+        await asyncio.wait_for(startup_responded.wait(), timeout=5)
+    finally:
+        task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await task
+    return messages_sent
+
+
+@pytest.mark.asyncio
+async def test_lifespan_startup_runs_before_any_request():
+    ds = Datasette(memory=True)
+    assert ds._startup_invoked is False
+    app = ds.app()
+
+    messages = await _drive_lifespan_startup(app)
+
+    assert {"type": "lifespan.startup.complete"} in messages
+    assert ds._startup_invoked is True
+    # Internal catalog tables should be populated too, entirely without an
+    # HTTP request having been made.
+    internal_db = ds.get_internal_database()
+    databases = await internal_db.execute("select * from catalog_databases")
+    assert len(databases.rows) >= 1
+
+
+@pytest.mark.asyncio
+async def test_lifespan_startup_failure_reports_lifespan_startup_failed():
+    class RaisingStartupPlugin:
+        __name__ = "RaisingStartupPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                raise RuntimeError("boom from startup hook")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(RaisingStartupPlugin(), name="raising_startup_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan_startup(app)
+    finally:
+        pm.unregister(name="raising_startup_plugin")
+
+    assert messages == [
+        {"type": "lifespan.startup.failed", "message": "boom from startup hook"}
+    ]
+    # The exception happened before invoke_startup() got to the end of its
+    # body, so startup is not considered to have completed.
+    assert ds._startup_invoked is False
+
+
+@pytest.mark.asyncio
+async def test_startup_runs_exactly_once_across_lifespan_and_first_request():
+    call_count = {"n": 0}
+
+    class CountingStartupPlugin:
+        __name__ = "CountingStartupPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                call_count["n"] += 1
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(CountingStartupPlugin(), name="counting_startup_plugin")
+    try:
+        # Build the ASGI app once, the way a real deployment does - and
+        # reuse the SAME app instance for both the lifespan drive and the
+        # HTTP requests below, since a fresh ds.app() call would reset the
+        # AsgiRunOnFirstRequest fallback's state.
+        app = ds.app()
+
+        messages = await _drive_lifespan_startup(app)
+        assert {"type": "lifespan.startup.complete"} in messages
+        assert call_count["n"] == 1
+
+        # A first HTTP request (as if the host never sent lifespan events,
+        # or lifespan already ran) should not run the hook again.
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            response1 = await client.get("/-/versions.json")
+            assert response1.status_code == 200
+            # ... nor should a second, repeat request.
+            response2 = await client.get("/-/versions.json")
+            assert response2.status_code == 200
+    finally:
+        pm.unregister(name="counting_startup_plugin")
+
+    assert call_count["n"] == 1
+
+
+@pytest.mark.asyncio
+async def test_no_lifespan_first_request_still_triggers_startup():
+    # Pin today's behavior: a client that never drives ASGI lifespan events
+    # at all (like httpx2.ASGITransport, which DatasetteClient uses) still
+    # gets startup armed by the AsgiRunOnFirstRequest fallback.
+    ds = Datasette(memory=True)
+    assert ds._startup_invoked is False
+    app = ds.app()
+    transport = httpx2.ASGITransport(app=app)
+    async with httpx2.AsyncClient(
+        transport=transport, base_url="http://localhost"
+    ) as client:
+        response = await client.get("/-/versions.json")
+        assert response.status_code == 200
+
+    assert ds._startup_invoked is True
+    internal_db = ds.get_internal_database()
+    databases = await internal_db.execute("select * from catalog_databases")
+    assert len(databases.rows) >= 1
+
+
+@pytest.mark.asyncio
+async def test_datasette_client_first_request_triggers_startup():
+    # Same as above, but through the real DatasetteClient (ds.client) that
+    # plugins and tests actually use, to confirm nothing regressed there.
+    ds = Datasette(memory=True)
+    assert ds._startup_invoked is False
+    response = await ds.client.get("/-/versions.json")
+    assert response.status_code == 200
+    assert ds._startup_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_concurrent_first_requests_all_wait_for_slow_startup():
+    call_count = {"n": 0}
+
+    class SlowStartupPlugin:
+        __name__ = "SlowStartupPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                call_count["n"] += 1
+                await asyncio.sleep(0.2)
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(SlowStartupPlugin(), name="slow_startup_plugin")
+    try:
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            responses = await asyncio.gather(
+                *[client.get("/-/versions.json") for _ in range(10)]
+            )
+    finally:
+        pm.unregister(name="slow_startup_plugin")
+
+    # Every one of the 10 simultaneous first requests must have blocked
+    # until startup actually finished, not raced ahead of it.
+    assert all(response.status_code == 200 for response in responses)
+    assert call_count["n"] == 1
+    assert ds._startup_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_setup_db_still_runs_when_invoke_startup_ran_first(tmp_path, monkeypatch):
+    # Regression test: `datasette serve` (cli.py _serve_async) calls
+    # ds.invoke_startup() directly, before uvicorn ever sends a
+    # lifespan.startup event that drives _startup_sequence(). If
+    # _startup_sequence()'s fast path only checked `_startup_invoked`, it
+    # would see startup already done and skip the immutable-database
+    # table-count precompute (setup_db) entirely - a silent regression
+    # versus main, where AsgiRunOnFirstRequest ran setup_db unconditionally
+    # on request #1.
+    db_path = tmp_path / "immutable.db"
+    conn = sqlite3.connect(str(db_path))
+    conn.execute("create table t (id integer primary key)")
+    conn.commit()
+    conn.close()
+
+    ds = Datasette([], immutables=[str(db_path)])
+
+    call_count = {"n": 0}
+    original_table_counts = Database.table_counts
+
+    async def counting_table_counts(self, *args, **kwargs):
+        call_count["n"] += 1
+        return await original_table_counts(self, *args, **kwargs)
+
+    monkeypatch.setattr(Database, "table_counts", counting_table_counts)
+
+    # Simulate the CLI path: invoke_startup() runs directly and completes
+    # BEFORE _startup_sequence() ever gets a chance to run setup_db.
+    await ds.invoke_startup()
+    assert ds._startup_invoked is True
+    assert call_count["n"] == 0
+
+    # The lifespan/first-request path (or the CLI itself, per the fix)
+    # calling the shared entry point afterwards must still precompute
+    # table counts for immutable databases.
+    await ds._startup_sequence()
+    assert call_count["n"] == 1
+    assert ds._setup_db_done is True
+
+    # Idempotency: a second call must not recompute.
+    await ds._startup_sequence()
+    assert call_count["n"] == 1
+
+
+@pytest.mark.asyncio
+async def test_asgi_wrapper_runs_after_startup_fallback_path():
+    class AssertStartupPlugin:
+        __name__ = "AssertStartupPlugin"
+
+        @hookimpl
+        def asgi_wrapper(self, datasette):
+            def wrap(app):
+                async def check_startup(scope, receive, send):
+                    if scope["type"] == "http":
+                        assert (
+                            datasette._startup_invoked is True
+                        ), "asgi_wrapper saw an http scope before startup completed"
+                    await app(scope, receive, send)
+
+                return check_startup
+
+            return wrap
+
+    ds = Datasette(memory=True)
+    pm.register(AssertStartupPlugin(), name="assert_startup_plugin")
+    try:
+        assert ds._startup_invoked is False
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            response = await client.get("/-/versions.json")
+            assert response.status_code == 200
+    finally:
+        pm.unregister(name="assert_startup_plugin")
+
+    assert ds._startup_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_short_circuit_wrapper_no_longer_defers_startup():
+    # Middleware that returns a response before getting to the rest of
+    # Datasette should still cause _startup_invoked=True
+    class ShortCircuitPlugin:
+        __name__ = "ShortCircuitPlugin"
+
+        @hookimpl
+        def asgi_wrapper(self, datasette):
+            def wrap(app):
+                async def forbidden(scope, receive, send):
+                    if scope["type"] != "http":
+                        await app(scope, receive, send)
+                        return
+                    await send(
+                        {
+                            "type": "http.response.start",
+                            "status": 403,
+                            "headers": [[b"content-type", b"text/plain"]],
+                        }
+                    )
+                    await send(
+                        {
+                            "type": "http.response.body",
+                            "body": b"Forbidden",
+                        }
+                    )
+
+                return forbidden
+
+            return wrap
+
+    ds = Datasette(memory=True)
+    pm.register(ShortCircuitPlugin(), name="short_circuit_plugin")
+    try:
+        assert ds._startup_invoked is False
+        app = ds.app()
+        transport = httpx2.ASGITransport(app=app)
+        async with httpx2.AsyncClient(
+            transport=transport, base_url="http://localhost"
+        ) as client:
+            response = await client.get("/-/versions.json")
+            assert response.status_code == 403
+    finally:
+        pm.unregister(name="short_circuit_plugin")
+
+    assert ds._startup_invoked is True

tests/test_load_extensions.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+from unittest import mock
 
 import pytest
 
@@ -20,6 +21,29 @@ def has_compiled_ext():
     return False
 
 
+@pytest.mark.parametrize("load_fails", (False, True))
+def test_load_extension_is_disabled(load_fails):
+    ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
+    connection = mock.Mock()
+    if load_fails:
+        connection.load_extension.side_effect = RuntimeError
+
+    if load_fails:
+        with pytest.raises(RuntimeError):
+            ds._prepare_connection(connection, "data")
+    else:
+        ds._prepare_connection(connection, "data")
+
+    # Extensions are loaded using the Python API, never via SQL
+    assert connection.load_extension.mock_calls == [
+        mock.call(COMPILED_EXTENSION_PATH),
+    ]
+    assert connection.enable_load_extension.mock_calls == [
+        mock.call(True),
+        mock.call(False),
+    ]
+
+
 @pytest.mark.asyncio
 @pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
 async def test_load_extension_default_entrypoint():
@@ -64,3 +88,20 @@ async def test_load_extension_multiple_entrypoints():
     response = await ds.client.get("/_memory/-/query.json?_shape=arrays&sql=select+c()")
     assert response.status_code == 200
     assert response.json()["rows"][0][0] == "c"
+
+
+@pytest.mark.asyncio
+@pytest.mark.skipif(not has_compiled_ext(), reason="Requires compiled ext.c")
+async def test_sql_cannot_load_additional_extension():
+    ds = Datasette(sqlite_extensions=[COMPILED_EXTENSION_PATH])
+
+    response = await ds.client.get(
+        "/_memory/-/query.json",
+        params={
+            "sql": "select load_extension(:path, :entrypoint)",
+            "path": COMPILED_EXTENSION_PATH,
+            "entrypoint": "sqlite3_ext_b_init",
+        },
+    )
+    assert response.status_code == 400
+    assert response.json()["error"] == "not authorized"

tests/test_permission_endpoints.py

@@ -494,3 +494,31 @@ async def test_execute_sql_requires_view_database():
         )
     finally:
         ds.pm.unregister(plugin)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("path", ["/-/allowed", "/-/allowed.json?action=view-table"])
+async def test_allowed_requires_view_instance(path):
+    """
+    GHSA-hp2x-vx2r-6vxg: /-/allowed should be gated like its /-/rules sibling.
+
+    An actor who is denied view-instance gets 403 from / and /-/rules, but
+    /-/allowed (HTML and JSON) currently returns 200 to the same actor.
+    """
+    ds = Datasette(config={"allow": {"id": "alice"}})
+    await ds.invoke_startup()
+    db = ds.add_memory_database("live")
+    await db.execute_write("CREATE TABLE IF NOT EXISTS t (id INTEGER PRIMARY KEY)")
+    await ds.refresh_schemas()
+
+    assert (await ds.client.get("/")).status_code == 403
+    assert (await ds.client.get("/-/rules.json?action=view-table")).status_code == 403
+
+    response = await ds.client.get(path)
+    assert response.status_code == 403
+
+    # Alice is still allowed
+    response = await ds.client.get(
+        path, cookies={"ds_actor": ds.client.actor_cookie({"id": "alice"})}
+    )
+    assert response.status_code == 200

tests/test_permissions.py

@@ -520,6 +520,7 @@ def view_instance_client():
         "/-/plugins",
         "/-/settings",
         "/-/threads",
+        "/-/tasks",
         "/-/databases",
         "/-/permissions",
         "/-/messages",

tests/test_playwright.py

@@ -5,7 +5,7 @@ import subprocess
 import sys
 import time
 
-import httpx
+import httpx2
 import pytest
 
 from datasette.fixtures import write_fixture_database
@@ -34,11 +34,11 @@ def wait_for_server(process, url, timeout=30):
                 f"stderr:\n{stderr}"
             )
         try:
-            response = httpx.get(url, timeout=1.0)
+            response = httpx2.get(url, timeout=1.0)
             if response.status_code < 500:
                 return
             last_error = f"HTTP {response.status_code}: {response.text[:200]}"
-        except httpx.HTTPError as ex:
+        except httpx2.HTTPError as ex:
             last_error = repr(ex)
         time.sleep(0.1)
     if process.poll() is None:
@@ -108,6 +108,11 @@ def write_playwright_database(db_path):
     conn = sqlite3.connect(db_path)
     try:
         conn.executescript("""
+        create table count_numbers (id integer primary key);
+        with recursive sequence(id) as (
+            select 1 union all select id + 1 from sequence where id < 10002
+        )
+        insert into count_numbers select id from sequence;
         create table projects (
             id integer primary key,
             title text not null,
@@ -336,7 +341,7 @@ def project_rows(datasette_server, **filters):
         "_shape": "objects",
         **{key: str(value) for key, value in filters.items()},
     }
-    response = httpx.get(f"{datasette_server}data/projects.json", params=params)
+    response = httpx2.get(f"{datasette_server}data/projects.json", params=params)
     response.raise_for_status()
     return response.json()["rows"]
 
@@ -348,7 +353,7 @@ def project_row(datasette_server, pk):
 
 
 def binary_file_blob(datasette_server, pk):
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/binary_files/{pk}.blob",
         params={"_blob_column": "data"},
     )
@@ -369,7 +374,7 @@ def bulk_default_rows(datasette_server, **filters):
         "_shape": "objects",
         **{key: str(value) for key, value in filters.items()},
     }
-    response = httpx.get(f"{datasette_server}data/bulk_defaults.json", params=params)
+    response = httpx2.get(f"{datasette_server}data/bulk_defaults.json", params=params)
     response.raise_for_status()
     return response.json()["rows"]
 
@@ -379,7 +384,7 @@ def upsert_item_rows(datasette_server, **filters):
         "_shape": "objects",
         **{key: str(value) for key, value in filters.items()},
     }
-    response = httpx.get(f"{datasette_server}data/upsert_items.json", params=params)
+    response = httpx2.get(f"{datasette_server}data/upsert_items.json", params=params)
     response.raise_for_status()
     return response.json()["rows"]
 
@@ -473,7 +478,7 @@ def test_create_table_flow(page, datasette_server):
     page.wait_for_url("**/data/playwright_created")
     assert "playwright_created" in page.locator("h1").inner_text()
 
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/playwright_created.json?_extra=columns,column_types"
     )
     response.raise_for_status()
@@ -487,7 +492,7 @@ def test_create_table_flow(page, datasette_server):
     assert data["column_types"] == {
         "metadata": {"type": "json", "config": None},
     }
-    schema_response = httpx.get(
+    schema_response = httpx2.get(
         f"{datasette_server}data/-/query.json",
         params={
             "sql": (
@@ -603,7 +608,7 @@ def test_create_table_from_data_flow(page, datasette_server):
     dialog.locator(".table-create-save").click()
     page.wait_for_url("**/data/playwright_from_data")
 
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/playwright_from_data.json?_shape=objects"
     )
     response.raise_for_status()
@@ -639,7 +644,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
     dialog.locator(".table-create-save").click()
     page.wait_for_url("**/data/playwright_numeric_blanks")
 
-    response = httpx.get(
+    response = httpx2.get(
         f"{datasette_server}data/playwright_numeric_blanks.json?_shape=objects"
     )
     response.raise_for_status()
@@ -648,7 +653,7 @@ def test_create_table_from_csv_keeps_numeric_type_when_values_are_blank(
         {"name": "B", "score": None},
     ]
 
-    schema_response = httpx.get(
+    schema_response = httpx2.get(
         f"{datasette_server}data/-/query.json",
         params={
             "sql": (
@@ -856,7 +861,7 @@ def test_alter_table_flow(page, datasette_server):
 
     columns = []
     for _ in range(20):
-        response = httpx.get(f"{datasette_server}data/projects.json?_extra=columns")
+        response = httpx2.get(f"{datasette_server}data/projects.json?_extra=columns")
         response.raise_for_status()
         columns = response.json()["columns"]
         if "status" in columns:
@@ -1604,3 +1609,45 @@ def test_delete_row_flow_removes_row(page, datasette_server):
     page.locator(".row-mutation-status", has_text="Deleted row 1").wait_for()
     page.locator('tr[data-row="1"]').wait_for(state="detached")
     assert project_rows(datasette_server, id=1) == []
+
+
+@pytest.mark.playwright
+def test_count_all(page, datasette_server):
+    page.goto(datasette_server + "data/count_numbers?id__gt=1&_sort=id")
+    assert page.locator(".table-count").inner_text() == "10,000+ rows"
+    with page.expect_response("**/count_numbers/-/count?*") as response:
+        page.get_by_role("button", name="count all", exact=True).click()
+    assert response.value.request.method == "POST"
+    assert response.value.request.post_data is None
+    assert "content-type" not in response.value.request.headers
+    assert response.value.json() == {"ok": True, "count": 10001}
+    page.wait_for_function(
+        'document.querySelector(".table-count").textContent === "10,001 rows"'
+    )
+    assert page.locator(".count-all").count() == 0
+    assert "id" in page.locator("h3").first.inner_text()
+
+
+@pytest.mark.playwright
+def test_count_all_error_retry(page, datasette_server):
+    page.goto(datasette_server + "data/count_numbers?id__gt=1")
+    page.route(
+        "**/count_numbers/-/count?*",
+        lambda route: route.fulfill(
+            status=400,
+            content_type="application/json",
+            body=json.dumps({"ok": False, "errors": ["Count query timed out"]}),
+        ),
+    )
+    button = page.get_by_role("button", name="count all", exact=True)
+    button.click()
+    page.wait_for_function(
+        'document.querySelector(".count-error").textContent === "Count query timed out"'
+    )
+    assert button.is_enabled()
+    page.unroute("**/count_numbers/-/count?*")
+    button.click()
+    page.wait_for_function(
+        'document.querySelector(".table-count").textContent === "10,001 rows"'
+    )
+    assert page.locator(".count-error").inner_text() == ""

tests/test_plugins.py

@@ -53,6 +53,13 @@ def test_hook_jump_items_sql():
     assert "jump_items_sql" in dir(pm.hook)
 
 
+def test_hook_shutdown():
+    # Detailed behavior (ordering against background-task cancellation and
+    # close(), idempotency, exception handling, sync vs async support) is
+    # covered in tests/test_shutdown.py.
+    assert "shutdown" in dir(pm.hook)
+
+
 @pytest.mark.asyncio
 async def test_hook_plugins_dir_plugin_prepare_connection(ds_client):
     response = await ds_client.get(
@@ -1300,7 +1307,8 @@ async def test_hook_filters_from_request(ds_client):
 
     ds_client.ds.pm.register(ReturnNothingPlugin(), name="ReturnNothingPlugin")
     response = await ds_client.get("/fixtures/facetable?_nothing=1")
-    assert "0 rows\n        where NOTHING" in response.text
+    summary = Soup(response.text, "html.parser").select_one(".table-summary")
+    assert summary.get_text(" ", strip=True) == "0 rows where NOTHING"
     json_response = await ds_client.get("/fixtures/facetable.json?_nothing=1")
     assert json_response.json()["rows"] == []
     ds_client.ds.pm.unregister(name="ReturnNothingPlugin")

tests/test_pr76_fts_policy.py

@@ -0,0 +1,140 @@
+"""Policy and compatibility coverage for PR #76, run against the fixed checkout."""
+
+import uuid
+
+import pytest
+
+from datasette.app import Datasette
+from datasette.resources import TableResource
+from datasette.utils.sqlite import sqlite3, sqlite_derived_table_dependencies
+
+
+@pytest.mark.parametrize("vocab_name", ["words", "name USING fts4aux", 'quoted"name'])
+@pytest.mark.parametrize(
+    "module,arguments",
+    [
+        ("fts5vocab", "'Search,Index', 'row'"),
+        ("fts5vocab", "'SEARCH,INDEX', 'col'"),
+        ("fts5vocab", "'Search,Index', 'instance'"),
+        ("fts4aux", "'Search,Index'"),
+    ],
+)
+def test_vocabulary_dependency_identity(module, arguments, vocab_name):
+    conn = sqlite3.connect(":memory:")
+    try:
+        fts = "fts5" if module == "fts5vocab" else "fts4"
+        conn.execute(f'create virtual table "Search,Index" using {fts}(body)')
+        quoted_name = '"' + vocab_name.replace('"', '""') + '"'
+        conn.execute(
+            f"create virtual table {quoted_name} USING /* module */ {module}({arguments})"
+        )
+        assert sqlite_derived_table_dependencies(conn)[vocab_name] == "Search,Index"
+    finally:
+        conn.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("module", ["fts5", "fts4"])
+@pytest.mark.parametrize("external_content", [False, True], ids=["one-hop", "two-hop"])
+@pytest.mark.parametrize(
+    "source_allowed,vocab_allowed", [(False, True), (True, False), (True, True)]
+)
+async def test_vocabulary_immediate_source_permissions(
+    module, external_content, source_allowed, vocab_allowed
+):
+    ds = Datasette(
+        memory=True,
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "search": {
+                            "permissions": {
+                                "view-table": (
+                                    {"id": "reader"} if source_allowed else False
+                                )
+                            }
+                        },
+                        "words": {"permissions": {"view-table": vocab_allowed}},
+                    }
+                }
+            }
+        },
+    )
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.execute_write("create table documents(body text)")
+    options = "body, content='documents'" if external_content else "body"
+    await db.execute_write(f"create virtual table search using {module}({options})")
+    definition = (
+        "fts5vocab('SEARCH', 'row')" if module == "fts5" else "fts4aux('SEARCH')"
+    )
+    await db.execute_write(f"create virtual table words using {definition}")
+    await ds.invoke_startup()
+    try:
+        actor = {"id": "reader"}
+        expected = source_allowed and vocab_allowed and not external_content
+        for name in ("words", "WORDS"):
+            assert (
+                await ds.allowed(
+                    action="view-table",
+                    resource=TableResource("data", name),
+                    actor=actor,
+                )
+                is expected
+            )
+        resources = await ds.allowed_resources(
+            "view-table", parent="data", actor=actor, include_is_private=True
+        )
+        words = [r for r in resources.resources if r.child == "words"]
+        assert bool(words) is expected
+        if expected:
+            assert words[0].private
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "words")
+        )
+        # Dropping the source invalidates dependency metadata and remains denied.
+        await db.execute_write("drop table search")
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "words"), actor=actor
+        )
+    finally:
+        ds.close()
+
+
+@pytest.mark.parametrize(
+    "module,definition",
+    [
+        ("fts5", "fts5vocab('main', 'search', 'row')"),
+        ("fts4", "fts4aux('main', 'search')"),
+    ],
+)
+def test_cross_schema_vocabulary_is_unresolved(module, definition):
+    conn = sqlite3.connect(":memory:")
+    try:
+        conn.execute(f"create virtual table search using {module}(body)")
+        conn.execute(f"create virtual table temp.words using {definition}")
+        # Cross-schema ownership is not representable by the current map.
+        # The source is itself derived, so the immediate-source policy denies it.
+        assert (
+            sqlite_derived_table_dependencies(conn, schema="temp")["words"] == "words"
+        )
+    finally:
+        conn.close()
+
+
+@pytest.mark.parametrize(
+    "definition",
+    [
+        """CREATE VIRTUAL TABLE"words"USING"fts5vocab"('search', 'row')""",
+        """CREATE VIRTUAL TABLE[words]USING[fts5vocab]('search', 'row')""",
+        """CREATE VIRTUAL TABLE`words`USING`fts5vocab`('search', 'row')""",
+    ],
+)
+def test_vocabulary_quoted_token_boundaries(definition):
+    conn = sqlite3.connect(":memory:")
+    try:
+        conn.execute("create virtual table search using fts5(body)")
+        conn.execute(definition)
+        assert sqlite_derived_table_dependencies(conn)["words"] == "search"
+    finally:
+        conn.close()

tests/test_pr76_statistics_policy.py

@@ -0,0 +1,113 @@
+"""Statistics access policy and plugin replacement coverage for PR #76."""
+
+import uuid
+
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.permissions import PermissionSQL
+from datasette.resources import TableResource
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("scope", [None, "global", "database", "table", "root"])
+async def test_statistics_denied_despite_allow_rules(scope):
+    config = {"databases": {"data": {"tables": {"sqlite_stat1": {}}}}}
+    grant = {"view-table": True}
+    if scope == "global":
+        config["permissions"] = grant
+    elif scope == "database":
+        config["databases"]["data"]["permissions"] = grant
+    elif scope == "table":
+        config["databases"]["data"]["tables"]["sqlite_stat1"]["permissions"] = grant
+    ds = Datasette(memory=True, config=config)
+    ds.root_enabled = scope == "root"
+    actor = {"id": "root"} if scope == "root" else {"id": "reader"}
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.execute_write("create table items(value text)")
+    await db.execute_write("create index items_value on items(value)")
+    await db.execute_write("insert into items values ('example')")
+    await db.execute_write("analyze")
+    await ds.invoke_startup()
+    try:
+        assert "view-sqlite-statistics" not in ds.actions
+        for name in ("sqlite_stat1", "SQLITE_STAT1"):
+            assert not await ds.allowed(
+                action="view-table", resource=TableResource("data", name), actor=actor
+            )
+        for suffix in ("", ".json", ".csv"):
+            assert (
+                await ds.client.get(f"/data/sqlite_stat1{suffix}", actor=actor)
+            ).status_code == 403
+        resources = await ds.allowed_resources("view-table", parent="data", actor=actor)
+        assert "sqlite_stat1" not in {r.child for r in resources.resources}
+        assert "items" in {r.child for r in resources.resources}
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "table", ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"]
+)
+@pytest.mark.parametrize("default_deny", [False, True])
+async def test_statistics_names_denied(table, default_deny):
+    ds = Datasette(memory=True, default_deny=default_deny)
+    ds.root_enabled = True
+    await ds.invoke_startup()
+    try:
+        for name in (table, table.upper()):
+            assert not await ds.allowed(
+                action="view-table",
+                resource=TableResource("_memory", name),
+                actor={"id": "root"},
+            )
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_plugin_can_replace_statistics_policy():
+    class ReplacementPolicy:
+        @hookimpl
+        def permission_resources_sql(self, action, actor):
+            if action == "view-table":
+                return PermissionSQL(
+                    sql="SELECT 'data' AS parent, 'sqlite_stat1' AS child, :statistics_allowed AS allow, 'custom statistics policy' AS reason",
+                    params={"statistics_allowed": int(actor == {"id": "reader"})},
+                )
+
+    ds = Datasette(memory=True)
+    db = ds.add_memory_database(uuid.uuid4().hex, name="data")
+    await db.execute_write("create table items(value text)")
+    await db.execute_write("analyze")
+    await ds.invoke_startup()
+    name = "datasette.default_permissions.sqlite_statistics"
+    original = ds.pm.unregister(name=name)
+    assert original is not None
+    replacement = ReplacementPolicy()
+    ds.pm.register(replacement, name="test-replacement-statistics-policy")
+    try:
+        actor = {"id": "reader"}
+        assert await ds.allowed(
+            action="view-table",
+            resource=TableResource("data", "sqlite_stat1"),
+            actor=actor,
+        )
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "sqlite_stat1")
+        )
+        resources = await ds.allowed_resources(
+            "view-table", parent="data", actor=actor, include_is_private=True
+        )
+        stats = [r for r in resources.resources if r.child == "sqlite_stat1"]
+        assert len(stats) == 1 and stats[0].private
+        assert (
+            await ds.client.get("/data/sqlite_stat1.json", actor=actor)
+        ).status_code == 200
+        assert (await ds.client.get("/data/sqlite_stat1.json")).status_code == 403
+    finally:
+        ds.pm.unregister(replacement)
+        ds.pm.register(original, name=name)
+        ds.close()

tests/test_queries.py

@@ -3248,74 +3248,6 @@ async def test_execute_write_create_table_uses_create_table_permission():
     assert not await db.table_exists("should_not_exist")
 
 
-@pytest.mark.asyncio
-async def test_execute_write_create_view_uses_create_view_permission():
-    ds = Datasette(
-        memory=True,
-        default_deny=True,
-        config={
-            "permissions": {
-                "insert-row": {"id": "row-writer"},
-                "update-row": {"id": "row-writer"},
-            },
-            "databases": {
-                "data": {
-                    "permissions": {
-                        "view-database": {"id": ["creator", "row-writer"]},
-                        "execute-write-sql": {"id": ["creator", "row-writer"]},
-                        "create-view": {"id": "creator"},
-                    }
-                }
-            },
-        },
-    )
-    db = ds.add_memory_database("execute_write_create_view", name="data")
-    await db.execute_write("create table dogs (id integer primary key, name text)")
-    await ds.invoke_startup()
-
-    analysis_response = await ds.client.get(
-        "/data/-/execute-write/analyze",
-        actor={"id": "creator"},
-        params={"sql": "create view dog_names as select id, name from dogs"},
-    )
-    allowed_response = await ds.client.post(
-        "/data/-/execute-write",
-        actor={"id": "creator"},
-        json={"sql": "create view dog_names as select id, name from dogs"},
-    )
-    row_permission_response = await ds.client.post(
-        "/data/-/execute-write",
-        actor={"id": "row-writer"},
-        json={"sql": "create view should_not_exist as select id from dogs"},
-    )
-
-    assert analysis_response.status_code == 200
-    analysis_data = analysis_response.json()
-    assert analysis_data["ok"] is True
-    assert analysis_data["execute_disabled"] is False
-    assert analysis_data["analysis_rows"] == [
-        {
-            "operation": "create",
-            "database": "data",
-            "table": "dog_names",
-            "required_permission": "create-view",
-            "source": None,
-            "allowed": True,
-        }
-    ]
-
-    assert allowed_response.status_code == 200
-    assert allowed_response.json()["ok"] is True
-    assert allowed_response.json()["message"] == "Query executed"
-    assert await db.view_exists("dog_names")
-
-    assert row_permission_response.status_code == 403
-    assert row_permission_response.json()["errors"] == [
-        "Permission denied: need create-view on data"
-    ]
-    assert not await db.view_exists("should_not_exist")
-
-
 @pytest.mark.parametrize(
     (
         "database_name",

tests/test_schema_endpoints.py

@@ -246,3 +246,114 @@ async def test_table_not_exists(schema_ds):
     response = await schema_ds.client.get("/schema_public_db/nonexistent/-/schema.md")
     assert response.status_code == 404
     assert "not found" in response.text.lower()
+
+
+@pytest_asyncio.fixture(scope="module")
+async def schema_table_perms_ds():
+    """
+    A database that is viewable by anonymous users, but with one table
+    locked down using the documented per-table lockdown recipe:
+    a table-level allow block combined with allow_sql: false.
+    """
+    ds = Datasette(
+        config={
+            "databases": {
+                "schema_table_perms_db": {
+                    "allow_sql": False,
+                    "tables": {"employee_salaries": {"allow": {"id": "root"}}},
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("schema_table_perms_db")
+    await db.execute_write(
+        "CREATE TABLE IF NOT EXISTS public_posts (id INTEGER PRIMARY KEY, title TEXT)"
+    )
+    await db.execute_write(
+        "CREATE TABLE IF NOT EXISTS employee_salaries "
+        "(id INTEGER PRIMARY KEY, ssn TEXT, salary_usd INTEGER)"
+    )
+    await db.execute_write(
+        "CREATE INDEX IF NOT EXISTS idx_employee_salaries_ssn ON employee_salaries(ssn)"
+    )
+    await db.execute_write(
+        "CREATE TRIGGER IF NOT EXISTS trg_employee_salaries "
+        "AFTER INSERT ON employee_salaries BEGIN SELECT 1; END"
+    )
+    return ds
+
+
+@pytest.mark.asyncio
+async def test_schema_table_perms_controls(schema_table_perms_ds):
+    """Sanity check: the locked down table really is denied to anonymous users."""
+    ds = schema_table_perms_ds
+    for path in (
+        "/schema_table_perms_db/employee_salaries.json",
+        "/schema_table_perms_db/employee_salaries/-/schema.json",
+        "/schema_table_perms_db/-/query.json?sql=select+*+from+employee_salaries",
+    ):
+        response = await ds.client.get(path)
+        assert response.status_code == 403, path
+    response = await ds.client.get("/schema_table_perms_db.json")
+    assert response.status_code == 200
+    assert "employee_salaries" not in response.text
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "base_url",
+    ["/-/schema", "/schema_table_perms_db/-/schema"],
+)
+@pytest.mark.parametrize("format_ext", ["json", "md", ""])
+async def test_schema_parent_views_hide_denied_tables(
+    schema_table_perms_ds, base_url, format_ext
+):
+    """
+    GHSA-926p-cw2f-643h: /-/schema and /db/-/schema must not disclose the DDL
+    of tables the actor is denied view-table on, including indexes and
+    triggers that belong to those tables.
+    """
+    url = base_url + (f".{format_ext}" if format_ext else "")
+
+    # Anonymous: allowed table visible, denied table (and its columns,
+    # index and trigger) absent
+    response = await schema_table_perms_ds.client.get(url)
+    assert response.status_code == 200
+    assert "public_posts" in response.text
+    assert "employee_salaries" not in response.text
+    assert "ssn" not in response.text
+    assert "salary_usd" not in response.text
+    assert "idx_employee_salaries_ssn" not in response.text
+    assert "trg_employee_salaries" not in response.text
+
+    # root can see everything
+    response = await schema_table_perms_ds.client.get(url, actor={"id": "root"})
+    assert response.status_code == 200
+    assert "public_posts" in response.text
+    assert "CREATE TABLE employee_salaries" in response.text
+    assert "idx_employee_salaries_ssn" in response.text
+    assert "trg_employee_salaries" in response.text
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "object_name", ["idx_employee_salaries_ssn", "trg_employee_salaries"]
+)
+@pytest.mark.parametrize("format_ext", ["json", "md", ""])
+async def test_table_schema_does_not_serve_objects_of_denied_table(
+    schema_table_perms_ds, object_name, format_ext
+):
+    """
+    Related to GHSA-926p-cw2f-643h: /db/<name>/-/schema looks up sqlite_master
+    by name without restricting to tables/views, so requesting the name of an
+    index or trigger that belongs to a denied table serves its DDL. The
+    view-table check runs against the index/trigger name, which is not a
+    restricted table, so it passes.
+    """
+    url = f"/schema_table_perms_db/{object_name}/-/schema"
+    if format_ext:
+        url += f".{format_ext}"
+    response = await schema_table_perms_ds.client.get(url)
+    assert response.status_code in (403, 404)
+    assert "employee_salaries" not in response.text
+    assert "ssn" not in response.text

tests/test_shutdown.py

@@ -0,0 +1,318 @@
+"""
+Tests for the shutdown(datasette) plugin hook and Datasette.invoke_shutdown().
+
+Order under test: plugin `shutdown` hooks run first (while background tasks
+are still alive) -> supervised background tasks are cancelled and drained
+(fixed 5s grace) -> databases are closed. Hook exceptions are logged, never
+propagated, and never skip a later step. The whole sequence is idempotent,
+so a second lifespan.shutdown message (or any other second caller) is a
+no-op.
+"""
+
+import asyncio
+import contextlib
+import logging
+
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.plugins import pm
+
+
+async def _drive_lifespan(app, messages):
+    """Drive a single ASGI lifespan connection against `app`, feeding
+    `messages` to it in order via receive(). Returns once `app` itself
+    returns, which - per AsgiLifespan - happens after it has processed a
+    lifespan.shutdown message and sent back .complete or .failed. Returns
+    the list of messages `app` sent via send().
+
+    `messages` must end with a lifespan.shutdown (or a startup that fails)
+    or this will hang forever waiting for a message that never comes,
+    since AsgiLifespan only returns after handling shutdown.
+    """
+    sent = []
+    idx = 0
+
+    async def receive():
+        nonlocal idx
+        if idx < len(messages):
+            message = messages[idx]
+            idx += 1
+            return message
+        # Real servers park here waiting for lifespan.shutdown; nothing
+        # left to deliver in this test, so just block - the caller is
+        # expected to have already gotten what it needs via a preceding
+        # lifespan.shutdown in `messages`.
+        await asyncio.Event().wait()
+
+    async def send(message):
+        sent.append(message)
+
+    await app({"type": "lifespan"}, receive, send)
+    return sent
+
+
+@pytest.mark.asyncio
+async def test_shutdown_hook_runs_before_task_cancellation_then_closes():
+    # Ordering proof: the shutdown hook observes the background task still
+    # "running" (hooks run BEFORE cancellation); after the whole lifespan
+    # drive completes, the task is "cancelled" and the Datasette instance
+    # is closed.
+    events = []
+    task_state_seen_in_hook = {}
+
+    async def bg_task(datasette):
+        await asyncio.Event().wait()
+
+    class LifecyclePlugin:
+        __name__ = "LifecyclePlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(bg_task, name="bg-task")
+
+            return inner
+
+        @hookimpl
+        def shutdown(self, datasette):
+            async def inner():
+                events.append("shutdown-hook-ran")
+                handle = datasette._background_tasks.tasks()[0]
+                task_state_seen_in_hook["state"] = handle.state
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(LifecyclePlugin(), name="lifecycle_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan(
+            app, [{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]
+        )
+    finally:
+        pm.unregister(name="lifecycle_plugin")
+
+    assert {"type": "lifespan.startup.complete"} in messages
+    assert {"type": "lifespan.shutdown.complete"} in messages
+    assert events == ["shutdown-hook-ran"]
+    assert task_state_seen_in_hook["state"] == "running"
+
+    handle = ds._background_tasks.tasks()[0]
+    assert handle.state == "cancelled"
+    assert ds._closed is True
+    assert ds._shutdown_invoked is True
+
+
+@pytest.mark.asyncio
+async def test_second_lifespan_shutdown_does_not_double_invoke():
+    call_count = {"n": 0}
+
+    class CountingShutdownPlugin:
+        __name__ = "CountingShutdownPlugin"
+
+        @hookimpl
+        def shutdown(self, datasette):
+            async def inner():
+                call_count["n"] += 1
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(CountingShutdownPlugin(), name="counting_shutdown_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan(
+            app, [{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]
+        )
+        assert {"type": "lifespan.shutdown.complete"} in messages
+        assert call_count["n"] == 1
+
+        # A second, separate lifespan connection (a misbehaving host, or a
+        # second embedder driving the same Datasette instance) sends
+        # lifespan.shutdown again. AsgiLifespan itself has no memory of
+        # the earlier connection, so this exercises invoke_shutdown()'s
+        # own idempotency guard, not anything ASGI-layer.
+        messages2 = await _drive_lifespan(app, [{"type": "lifespan.shutdown"}])
+        assert {"type": "lifespan.shutdown.complete"} in messages2
+        assert call_count["n"] == 1
+    finally:
+        pm.unregister(name="counting_shutdown_plugin")
+
+    assert ds._closed is True
+
+
+@pytest.mark.asyncio
+async def test_raising_shutdown_hook_is_logged_and_does_not_block_the_rest(caplog):
+    events = []
+
+    async def bg_task(datasette):
+        await asyncio.Event().wait()
+
+    class RaisingShutdownPlugin:
+        __name__ = "RaisingShutdownPlugin"
+
+        @hookimpl
+        def shutdown(self, datasette):
+            async def inner():
+                raise RuntimeError("boom from shutdown hook")
+
+            return inner
+
+    class WellBehavedPlugin:
+        __name__ = "WellBehavedPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(bg_task, name="bg-task")
+
+            return inner
+
+        @hookimpl
+        def shutdown(self, datasette):
+            async def inner():
+                events.append("well-behaved-ran")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(RaisingShutdownPlugin(), name="raising_shutdown_plugin")
+    pm.register(WellBehavedPlugin(), name="well_behaved_plugin")
+    try:
+        app = ds.app()
+        with caplog.at_level(logging.ERROR, logger="datasette"):
+            messages = await _drive_lifespan(
+                app, [{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]
+            )
+    finally:
+        pm.unregister(name="raising_shutdown_plugin")
+        pm.unregister(name="well_behaved_plugin")
+
+    # The exception must never turn into lifespan.shutdown.failed - it's
+    # swallowed inside invoke_shutdown(), logged, and teardown continues.
+    assert {"type": "lifespan.shutdown.complete"} in messages
+    assert events == ["well-behaved-ran"]
+    assert "shutdown hook failed" in caplog.text
+    assert "boom from shutdown hook" in caplog.text
+
+    handle = ds._background_tasks.tasks()[0]
+    assert handle.state == "cancelled"
+    assert ds._closed is True
+
+
+@pytest.mark.asyncio
+async def test_sync_shutdown_hook_variant_works():
+    events = []
+
+    class SyncShutdownPlugin:
+        __name__ = "SyncShutdownPlugin"
+
+        @hookimpl
+        def shutdown(self, datasette):
+            # Deliberately not returning a coroutine/callable - a plain
+            # sync hookimpl, same as `def startup(datasette): ...` is
+            # supported via await_me_maybe.
+            events.append("sync-shutdown-ran")
+
+    ds = Datasette(memory=True)
+    pm.register(SyncShutdownPlugin(), name="sync_shutdown_plugin")
+    try:
+        app = ds.app()
+        messages = await _drive_lifespan(
+            app, [{"type": "lifespan.startup"}, {"type": "lifespan.shutdown"}]
+        )
+    finally:
+        pm.unregister(name="sync_shutdown_plugin")
+
+    assert {"type": "lifespan.shutdown.complete"} in messages
+    assert events == ["sync-shutdown-ran"]
+    assert ds._closed is True
+
+
+@pytest.mark.asyncio
+async def test_shutdown_logs_stragglers_that_outlive_the_grace_period(
+    caplog, monkeypatch
+):
+    async def stubborn(datasette):
+        with contextlib.suppress(asyncio.CancelledError):
+            await asyncio.sleep(10)
+        # Swallowing CancelledError above and returning normally simulates
+        # a task that ignores cancellation for longer than the grace
+        # period - same shape as test_background_tasks.py's equivalent
+        # test.
+        await asyncio.sleep(10)
+
+    class StubbornTaskPlugin:
+        __name__ = "StubbornTaskPlugin"
+
+        @hookimpl
+        def startup(self, datasette):
+            async def inner():
+                datasette.add_background_task(stubborn, name="stubborn-task")
+
+            return inner
+
+    ds = Datasette(memory=True)
+    pm.register(StubbornTaskPlugin(), name="stubborn_task_plugin")
+
+    # invoke_shutdown() calls self._background_tasks.cancel_all(grace=5.0)
+    # with a grace hardcoded in app.py, per the ticket. Rather than
+    # sleeping for a real 5s in this test, monkeypatch the
+    # BackgroundTaskSupervisor *instance's* cancel_all to a wrapper that
+    # ignores the caller-supplied grace and substitutes a small one - this
+    # is the cleanest seam because it requires no production-code changes
+    # (no grace= setting/attribute to add) and leaves invoke_shutdown's
+    # own code under test untouched.
+    real_cancel_all = ds._background_tasks.cancel_all
+
+    async def fast_cancel_all(grace=5.0):
+        return await real_cancel_all(grace=0.1)
+
+    monkeypatch.setattr(ds._background_tasks, "cancel_all", fast_cancel_all)
+
+    try:
+        app = ds.app()
+
+        sent = []
+        queue = asyncio.Queue()
+        startup_complete = asyncio.Event()
+
+        async def receive():
+            return await queue.get()
+
+        async def send(message):
+            sent.append(message)
+            if message.get("type") == "lifespan.startup.complete":
+                startup_complete.set()
+
+        task = asyncio.create_task(app({"type": "lifespan"}, receive, send))
+        await queue.put({"type": "lifespan.startup"})
+        await asyncio.wait_for(startup_complete.wait(), timeout=5)
+
+        # Let the stubborn background task actually start running and
+        # reach its CancelledError-suppressing sleep before shutdown
+        # cancels it - a task cancelled before it has ever run its first
+        # step never enters that block at all, so it would finish
+        # cancelling immediately instead of behaving like a straggler.
+        await asyncio.sleep(0.05)
+
+        with caplog.at_level(logging.WARNING, logger="datasette.background_tasks"):
+            await queue.put({"type": "lifespan.shutdown"})
+            await asyncio.wait_for(task, timeout=5)
+
+        assert {"type": "lifespan.shutdown.complete"} in sent
+        assert "stubborn-task" in caplog.text
+        assert ds._closed is True
+    finally:
+        pm.unregister(name="stubborn_task_plugin")
+        # Clean up: the stubborn task ignored cancellation and is still
+        # sleeping past the shrunk grace period; actually cancel and await
+        # it now that the test has made its assertions, so it doesn't leak
+        # past the end of the test.
+        handles = ds._background_tasks.tasks()
+        if handles and handles[0].task and not handles[0].task.done():
+            handles[0].task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await handles[0].task

tests/test_stored_queries.py

@@ -208,11 +208,12 @@ def test_custom_params(stored_write_client):
     )
 
 
-def test_stored_query_pages_no_vary_header(stored_write_client):
+def test_stored_query_pages_vary_by_credentials(stored_write_client):
-    # These pages no longer embed per-cookie CSRF tokens, so they must not
+    # Even without per-cookie CSRF tokens, anonymous pages must not be reused
-    # set Vary: Cookie - they should be cacheable across users.
+    # for authenticated users whose permissions or navigation can differ.
-    assert "vary" not in stored_write_client.get("/data").headers
+    for path in ("/data", "/data/update_name"):
-    assert "vary" not in stored_write_client.get("/data/update_name").headers
+        response = stored_write_client.get(path)
+        assert response.headers["vary"] == "Cookie, Authorization"
 
 
 def test_json_post_body(stored_write_client):

tests/test_success_envelope.py

@@ -59,6 +59,7 @@ async def test_success_object_has_ok_true(ds_client, path):
         "/-/rules.json?action=view-instance",
         "/-/check.json?action=view-instance",
         "/-/threads.json",
+        "/-/tasks.json",
     ),
 )
 async def test_permission_debug_success_has_ok_true(ds_envelope, path):

tests/test_table_api.py

@@ -619,7 +619,10 @@ def test_searchmode(table_metadata, querystring, expected_rows):
             ],
         ),
         (
-            "/fixtures/searchable_view.json?_shape=arrays&_search=weasel&_fts_table=searchable_fts&_fts_pk=pk",
+            (
+                "/fixtures/searchable_view_configured_by_metadata.json"
+                "?_shape=arrays&_search=weasel&_fts_table=searchable_fts&_fts_pk=pk"
+            ),
             [[2, "terry dog", "sara weasel", "puma"]],
         ),
     ],
@@ -1778,3 +1781,34 @@ async def test_next_url_included_by_default(ds_client):
         data = response.json()
     assert data["next"] is None
     assert data["next_url"] is None
+
+
+@pytest.mark.asyncio
+async def test_table_through_requires_view_table_on_through_table():
+    # GHSA-53fc-rhfg-h7qp issue 3: ?_through= runs a sub-select against the
+    # caller-supplied through table, so the actor must be allowed to view it.
+    # Otherwise it is an equality oracle over any column of a denied table.
+    from datasette.app import Datasette
+
+    ds = Datasette(
+        memory=True,
+        config={"databases": {"data": {"tables": {"salaries": {"allow": False}}}}},
+    )
+    db = ds.add_memory_database("table_through_denied", name="data")
+    await db.execute_write("create table people (id integer primary key, name text)")
+    await db.execute_write(
+        "create table salaries (id integer primary key, "
+        "person_id integer references people(id), note text)"
+    )
+    await db.execute_write("insert into people values (1, 'alice'), (2, 'bob')")
+    await db.execute_write("insert into salaries values (1, 1, 'TOPSECRET-A')")
+    await ds.invoke_startup()
+
+    # Sanity: anonymous cannot read salaries directly
+    assert (await ds.client.get("/data/salaries.json")).status_code == 403
+
+    response = await ds.client.get(
+        "/data/people.json?_shape=array"
+        '&_through={"table":"salaries","column":"note","value":"TOPSECRET-A"}'
+    )
+    assert response.status_code == 403, response.text

tests/test_table_html.py

@@ -270,7 +270,8 @@ async def test_empty_search_parameter_gets_removed(ds_client):
 async def test_searchable_view_persists_fts_table(ds_client):
     # The search form should persist ?_fts_table as a hidden field
     response = await ds_client.get(
-        "/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk"
+        "/fixtures/searchable_view_configured_by_metadata"
+        "?_fts_table=searchable_fts&_fts_pk=pk"
     )
     inputs = Soup(response.text, "html.parser").find("form").find_all("input")
     hiddens = [i for i in inputs if i["type"] == "hidden"]

tests/test_table_resource_identity.py

@@ -0,0 +1,420 @@
+"""Table permission identities must agree with SQLite identifier resolution."""
+
+import uuid
+from unittest.mock import AsyncMock
+
+import pytest
+
+from datasette import hookimpl
+from datasette.app import Datasette
+from datasette.default_permissions import restrictions_allow_action
+from datasette.permissions import Action, PermissionSQL, _permission_check_cache
+from datasette.resources import QueryResource, TableResource
+from datasette.utils.actions_sql import explain_permission_for_resource
+from datasette.utils.asgi import Forbidden
+from datasette.utils.permissions import gather_permission_sql_from_hooks
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("kind", ["table", "view"])
+@pytest.mark.parametrize("spelling", ["Inventory", "inventory", "INVENTORY"])
+@pytest.mark.parametrize("allowed", [False, True])
+@pytest.mark.parametrize("rule_spelling", ["Inventory", "iNvEnToRy"])
+async def test_table_permission_identity(
+    kind, spelling, allowed, rule_spelling, monkeypatch
+):
+    ds = Datasette(
+        config={
+            "permissions": {"view-table": not allowed, "insert-row": not allowed},
+            "databases": {
+                "data": {
+                    "tables": {
+                        rule_spelling: {
+                            "permissions": {
+                                "view-table": allowed,
+                                "insert-row": allowed,
+                            }
+                        }
+                    }
+                }
+            },
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    cache_token = _permission_check_cache.set({})
+    try:
+        await db.execute_write(
+            "create table Inventory (id integer primary key)"
+            if kind == "table"
+            else "create view Inventory as select 1 as id"
+        )
+        await ds.invoke_startup()
+        # Identity matching needs no target-schema lookup. Derived-table
+        # permissions may still check the schema version. All spellings and
+        # API entry points should share the existing permission result cache.
+        target_execute = AsyncMock(wraps=db.execute)
+        monkeypatch.setattr(db, "execute", target_execute)
+        internal_execute = AsyncMock(wraps=ds.get_internal_database().execute)
+        monkeypatch.setattr(ds.get_internal_database(), "execute", internal_execute)
+        resource = TableResource("data", spelling)
+        assert await ds.allowed_many(
+            actions=["view-table", "insert-row"], resource=resource
+        ) == {"view-table": allowed, "insert-row": allowed}
+        assert await ds.allowed(action="view-table", resource=resource) is allowed
+        assert await ds.check_visibility(None, "view-table", resource) == (
+            allowed,
+            False,
+        )
+        if allowed:
+            await ds.ensure_permission(action="view-table", resource=resource)
+        else:
+            with pytest.raises(Forbidden):
+                await ds.ensure_permission(action="view-table", resource=resource)
+        assert resource.child == spelling  # Do not mutate caller-owned resources.
+        for variant in ("Inventory", "inventory", "INVENTORY"):
+            assert (
+                await ds.allowed(
+                    action="view-table", resource=TableResource("data", variant)
+                )
+                is allowed
+            )
+        assert internal_execute.await_count == 1
+        assert all(
+            call.args[0] == "PRAGMA schema_version"
+            for call in target_execute.await_args_list
+        )
+        assert all(key[3] == "inventory" for key in _permission_check_cache.get())
+    finally:
+        _permission_check_cache.reset(cache_token)
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_other_permission_identities_are_preserved():
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "Äpfel": {"permissions": {"view-table": False}},
+                        "Future": {"permissions": {"view-table": False}},
+                    },
+                    "queries": {
+                        "Report": {
+                            "sql": "select 1",
+                            "permissions": {"view-query": False},
+                        },
+                        "report": {
+                            "sql": "select 1",
+                            "permissions": {"view-query": True},
+                        },
+                    },
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write('create table "Äpfel" (id integer primary key)')
+        await db.execute_write('create table "äpfel" (id integer primary key)')
+        await db.execute_write("create table Report (id integer primary key)")
+        await ds.invoke_startup()
+        # SQLite folds ASCII identifier casing, not Unicode casing.
+        for name, expected in [
+            ("ÄPFEL", False),
+            ("äPFEL", True),
+            ("Future", False),
+            ("future", False),
+        ]:
+            assert (
+                await ds.allowed(
+                    action="view-table", resource=TableResource("data", name)
+                )
+                is expected
+            )
+        # Query names remain case-sensitive even when a table has the same name.
+        for name, expected in [("Report", False), ("report", True)]:
+            assert (
+                await ds.allowed(
+                    action="view-query", resource=QueryResource("data", name)
+                )
+                is expected
+            )
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("allow", [True, False, {"id": "reader"}])
+async def test_table_listings_and_explanations(allow):
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        "inventory": {"permissions": {"view-table": allow}},
+                    }
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await db.execute_write("create view InventoryView as select id from Inventory")
+        await ds.invoke_startup()
+        for actor in (None, {"id": "reader"}):
+            expected = allow is True or (isinstance(allow, dict) and actor == allow)
+            explanation = await explain_permission_for_resource(
+                datasette=ds,
+                actor=actor,
+                action="view-table",
+                parent="data",
+                child="INVENTORY",
+            )
+            assert explanation["allowed"] is expected
+            assert explanation["winning_scope"] == "resource"
+            assert any(
+                "data/inventory" in rule["reason"]
+                for rule in explanation["matched_rules"]
+            )
+            page = await ds.allowed_resources(
+                "view-table",
+                actor,
+                parent="data",
+                include_is_private=True,
+                include_reasons=True,
+                limit=1,
+            )
+            resources = [resource async for resource in page.all()]
+            matching = [r for r in resources if r.child == "Inventory"]
+            assert bool(matching) is expected
+            assert len(matching) <= 1
+            if matching:
+                assert matching[0].private is isinstance(allow, dict)
+            assert any(r.child == "InventoryView" for r in resources)
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("deny_first", [True, False])
+async def test_case_variant_rules_deny_wins(deny_first):
+    rules = [("inventory", False), ("INVENTORY", True)]
+    if not deny_first:
+        rules.reverse()
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "tables": {
+                        name: {"permissions": {"view-table": allow}}
+                        for name, allow in rules
+                    }
+                }
+            }
+        }
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await ds.invoke_startup()
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "Inventory")
+        )
+        assert not (
+            await ds.allowed_resources(
+                "view-table", parent="data", include_is_private=True
+            )
+        ).resources
+        explanation = await explain_permission_for_resource(
+            datasette=ds,
+            actor=None,
+            action="view-table",
+            parent="data",
+            child="Inventory",
+        )
+        assert not explanation["allowed"]
+        assert any(
+            rule["effect"] == "allow" and not rule["decisive"]
+            for rule in explanation["matched_rules"]
+        )
+        assert any(
+            rule["effect"] == "deny" and rule["decisive"]
+            for rule in explanation["matched_rules"]
+        )
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("config_style", ["allow", "permissions"])
+@pytest.mark.parametrize("allowed", [True, False])
+async def test_case_variant_token_restrictions(config_style, allowed):
+    table_config = (
+        {"allow": allowed}
+        if config_style == "allow"
+        else {"permissions": {"view-table": allowed}}
+    )
+    ds = Datasette(
+        config={"databases": {"data": {"tables": {"Inventory": table_config}}}}
+    )
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    actor = {"id": "reader", "_r": {"r": {"data": {"inventory": ["vt"]}}}}
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await ds.invoke_startup()
+        assert restrictions_allow_action(
+            ds, actor["_r"], "view-table", ("data", "INVENTORY")
+        )
+        assert not restrictions_allow_action(
+            ds, actor["_r"], "view-table", ("Data", "Inventory")
+        )
+        assert (
+            await ds.allowed(
+                action="view-table",
+                resource=TableResource("data", "INVENTORY"),
+                actor=actor,
+            )
+            is allowed
+        )
+        page = await ds.allowed_resources("view-table", actor, parent="data")
+        assert [(r.parent, r.child) for r in page.resources] == (
+            [("data", "Inventory")] if allowed else []
+        )
+        explanation = await explain_permission_for_resource(
+            datasette=ds,
+            actor=actor,
+            action="view-table",
+            parent="data",
+            child="Inventory",
+        )
+        assert explanation["restriction_allowed"]
+        assert explanation["allowed"] is allowed
+    finally:
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_plugin_restriction_intersection_and_dependencies():
+    class Plugin:
+        @hookimpl
+        def register_actions(self, datasette):
+            return [
+                Action(
+                    name="inspect-inventory",
+                    description="Inspect inventory",
+                    resource_class=TableResource,
+                    also_requires="view-table",
+                )
+            ]
+
+        @hookimpl
+        def permission_resources_sql(self, action):
+            if action not in ("view-table", "inspect-inventory"):
+                return None
+            return [
+                PermissionSQL(
+                    sql="SELECT 'data' AS parent, 'INVENTORY' AS child, 1 AS allow, 'inventory grant' AS reason",
+                    restriction_sql="SELECT 'data' AS parent, 'inventory' AS child",
+                ),
+                PermissionSQL(
+                    restriction_sql="SELECT 'data' AS parent, 'InVeNtOrY' AS child"
+                ),
+            ]
+
+    ds = Datasette(default_deny=True)
+    ds.pm.register(Plugin(), name="identity-test")
+    db = ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    try:
+        await db.execute_write("create table Inventory (id integer primary key)")
+        await db.execute_write("create table Other (id integer primary key)")
+        await ds.invoke_startup()
+        for action in ("view-table", "inspect-inventory"):
+            assert await ds.allowed(
+                action=action, resource=TableResource("data", "Inventory")
+            )
+            assert not await ds.allowed(
+                action=action, resource=TableResource("data", "Other")
+            )
+            resources = (
+                await ds.allowed_resources(
+                    action, parent="data", include_is_private=True
+                )
+            ).resources
+            assert [r.child for r in resources] == ["Inventory"]
+            explanation = await explain_permission_for_resource(
+                datasette=ds,
+                actor=None,
+                action=action,
+                parent="data",
+                child="Inventory",
+            )
+            assert explanation["allowed"]
+            assert all(item["allowed"] for item in explanation["restrictions"])
+    finally:
+        ds.pm.unregister(name="identity-test")
+        ds.close()
+
+
+@pytest.mark.asyncio
+async def test_shared_plugin_rule_keeps_query_identity_and_original_sql():
+    shared = PermissionSQL(
+        sql="SELECT 'data' AS parent, 'Inventory' AS child, 0 AS allow, 'shared deny' AS reason"
+    )
+    original_sql = shared.sql
+
+    class Plugin:
+        @hookimpl
+        def permission_resources_sql(self, action):
+            if action in ("view-table", "view-query"):
+                return shared
+
+    ds = Datasette(
+        config={
+            "databases": {
+                "data": {
+                    "queries": {
+                        "Inventory": "select 1",
+                        "inventory": "select 1",
+                    }
+                }
+            }
+        }
+    )
+    ds.add_memory_database("identity_" + uuid.uuid4().hex, name="data")
+    ds.pm.register(Plugin(), name="identity-test")
+    try:
+        await ds.invoke_startup()
+        for _ in range(2):
+            await gather_permission_sql_from_hooks(
+                datasette=ds, actor=None, action="view-table"
+            )
+        assert shared.sql == original_sql
+        assert not await ds.allowed(
+            action="view-table", resource=TableResource("data", "inventory")
+        )
+        assert await ds.allowed(
+            action="view-query", resource=QueryResource("data", "inventory")
+        )
+        assert not await ds.allowed(
+            action="view-query", resource=QueryResource("data", "Inventory")
+        )
+        assert await ds.allowed(
+            action="view-table", resource=TableResource("Data", "Inventory")
+        )
+        assert restrictions_allow_action(
+            ds,
+            {"r": {"data": {"Inventory": ["vq"]}}},
+            "view-query",
+            ("data", "Inventory"),
+        )
+        assert not restrictions_allow_action(
+            ds,
+            {"r": {"data": {"Inventory": ["vq"]}}},
+            "view-query",
+            ("data", "inventory"),
+        )
+    finally:
+        ds.pm.unregister(name="identity-test")
+        ds.close()

tests/test_tasks_endpoint.py

@@ -0,0 +1,122 @@
+"""
+Tests for the /-/tasks introspection endpoint.
+
+/-/tasks exposes datasette._background_tasks (see tests/test_background_tasks.py
+for the supervisor machinery itself) the same way /-/threads exposes threading
+internals: gated behind the permissions-debug permission, JSON-only.
+"""
+
+import asyncio
+import contextlib
+import functools
+
+import pytest
+
+from datasette.app import Datasette
+
+
+async def example_task(datasette):
+    pass
+
+
+class ExampleWorker:
+    async def run(self, datasette):
+        pass
+
+    async def __call__(self, datasette):
+        pass
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "func, qualified_name",
+    [
+        (example_task, "example_task"),
+        (functools.partial(example_task), "example_task"),
+        (ExampleWorker().run, "ExampleWorker.run"),
+        (ExampleWorker(), "ExampleWorker.__call__"),
+    ],
+)
+async def test_task_function_path(func, qualified_name):
+    ds = Datasette(memory=True)
+    ds.root_enabled = True
+    handle = ds.add_background_task(func, name="custom-name")
+    try:
+        response = await ds.client.get("/-/tasks.json", actor={"id": "root"})
+        assert response.status_code == 200
+        task = response.json()["tasks"][0]
+        assert task["name"] == "custom-name"
+        assert task["function"] == f"{__name__}.{qualified_name}"
+        assert handle.function == task["function"]
+        assert "plugin" not in task
+        await handle.task
+        html = await ds.client.get("/-/tasks", actor={"id": "root"})
+        assert html.status_code == 200
+        assert task["function"] in html.text
+    finally:
+        await ds.invoke_shutdown()
+
+
+@pytest.mark.asyncio
+async def test_tasks_requires_permissions_debug():
+    ds = Datasette(memory=True)
+    ds.root_enabled = True
+
+    denied = await ds.client.get("/-/tasks.json")
+    assert denied.status_code == 403
+
+    allowed = await ds.client.get("/-/tasks.json", actor={"id": "root"})
+    assert allowed.status_code == 200
+    data = allowed.json()
+    assert data["ok"] is True
+    assert "tasks" in data
+    assert "launched" in data
+
+
+@pytest.mark.asyncio
+async def test_running_and_crashed_task_states():
+    ds = Datasette(memory=True)
+    ds.root_enabled = True
+
+    async def long_running(datasette):
+        await asyncio.Event().wait()
+
+    async def crashing_task(datasette):
+        raise RuntimeError("kaboom")
+
+    long_handle = ds.add_background_task(long_running, name="long-runner")
+    crash_handle = ds.add_background_task(crashing_task, name="crashing_task")
+
+    await ds.start_background_tasks()
+
+    # Let the crashing_task run to completion and its done-callback (which sets
+    # handle.state = "crashed") actually fire before we read state back out.
+    await asyncio.wait_for(
+        asyncio.gather(crash_handle.task, return_exceptions=True), timeout=5
+    )
+    await asyncio.sleep(0)
+
+    try:
+        response = await ds.client.get("/-/tasks.json", actor={"id": "root"})
+        assert response.status_code == 200
+        data = response.json()
+        assert data["launched"] is True
+
+        by_name = {t["name"]: t for t in data["tasks"]}
+        assert by_name["long-runner"]["state"] == "running"
+        assert by_name["long-runner"]["exception"] is None
+        assert by_name["long-runner"]["started_at"] is not None
+
+        crashed = by_name["crashing_task"]
+        assert crashed["function"] == (
+            f"{__name__}.test_running_and_crashed_task_states.<locals>.crashing_task"
+        )
+        assert crashed["state"] == "crashed"
+        assert crashed["exception"] is not None
+        assert isinstance(crashed["exception"], str)
+        assert "kaboom" in crashed["exception"]
+        assert "RuntimeError" in crashed["exception"]
+    finally:
+        long_handle.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await long_handle.task

tests/test_utils.py

@@ -16,6 +16,7 @@ from datasette.app import Datasette
 from datasette.utils.asgi import Request
 from datasette.utils.sqlite import (
     sqlite3,
+    sqlite_derived_table_dependencies,
     sqlite_hidden_table_names,
     sqlite_table_type,
     supports_returning,
@@ -369,6 +370,46 @@ def test_sqlite_hidden_table_names_hides_multiline_content_fts_table():
         conn.close()
 
 
+def test_sqlite_derived_table_dependencies():
+    conn = utils.sqlite3.connect(":memory:")
+    try:
+        conn.executescript("""
+            create table docs(id integer primary key, body text);
+            create virtual table external_fts5 using fts5(
+                body, content='docs', content_rowid='id'
+            );
+            create virtual table internal_fts5 using fts5(body);
+            create virtual table contentless_fts5 using fts5(body, content='');
+            create virtual table external_fts4 using fts4(body, content="docs");
+            create virtual table internal_fts4 using fts4(body);
+            create virtual table contentless_fts4 using fts4(body, content="");
+            create table [docs, archive](body text);
+            create virtual table commented_fts5 using fts5(
+                body, tokenize='porter unicode61',
+                /* Comments and commas in quoted values must not confuse parsing. */
+                content='docs, archive'
+            );
+            create virtual table boxes using rtree(id, minx, maxx, miny, maxy);
+        """)
+
+        dependencies = sqlite_derived_table_dependencies(conn)
+
+        assert dependencies["external_fts5"] == "docs"
+        assert dependencies["external_fts4"] == "docs"
+        assert dependencies["commented_fts5"] == "docs, archive"
+        assert "contentless_fts5" not in dependencies
+        assert "contentless_fts4" not in dependencies
+        assert dependencies["internal_fts5_content"] == "internal_fts5"
+        assert dependencies["internal_fts4_content"] == "internal_fts4"
+        assert dependencies["external_fts5_data"] == "external_fts5"
+        assert dependencies["external_fts4_segments"] == "external_fts4"
+        assert dependencies["boxes_node"] == "boxes"
+        assert dependencies["boxes_parent"] == "boxes"
+        assert dependencies["boxes_rowid"] == "boxes"
+    finally:
+        conn.close()
+
+
 @pytest.mark.parametrize(
     "url,expected",
     [

tests/test_utils_sql_analysis.py

@@ -439,7 +439,7 @@ def test_analyze_attached_database_tables(conn):
     }
 
 
-def test_analyze_clears_authorizer_on_error():
+def test_analyze_disables_authorizer_on_error():
     class FakeConnection:
         def __init__(self):
             self.authorizers = []
@@ -455,4 +455,5 @@ def test_analyze_clears_authorizer_on_error():
     with pytest.raises(sqlite3.OperationalError):
         analyze_sql_tables(conn, "bad SQL")
 
-    assert conn.authorizers[-1] is None
+    final_authorizer = conn.authorizers[-1]
+    assert final_authorizer is None or final_authorizer() == sqlite3.SQLITE_OK