.github/workflows/deploy-latest.yml

@@ -14,46 +14,24 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - name: Check deployment prerequisites
-      id: deployment-prerequisites
-      env:
-        GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
-        LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
-      run: |
-        missing=()
-        for variable in GCP_SA_KEY LATEST_DATASETTE_SECRET; do
-          if [[ -z "${!variable:-}" ]]; then
-            missing+=("$variable")
-          fi
-        done
-        if (( ${#missing[@]} )); then
-          echo "::notice::Skipping deployment because required environment variables are missing: ${missing[*]}"
-          echo "available=false" >> "$GITHUB_OUTPUT"
-        else
-          echo "available=true" >> "$GITHUB_OUTPUT"
-        fi
     - name: Check out datasette
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/checkout@v7
     - name: Set up Python
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: actions/setup-python@v6
       with:
         python-version: "3.13"
         cache: pip
     - name: Install Python dependencies
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         python -m pip install --upgrade pip
         python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
+        python -m pip install sphinx-to-sqlite==0.1a1
     - name: Run tests
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
+      if: ${{ github.ref == 'refs/heads/main' }}
       run: |
         pytest -n auto -m "not serial"
         pytest -m "serial"
     - name: Build fixtures.db and other files needed to deploy the demo
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |-
         python tests/fixtures.py \
           fixtures.db \
@@ -61,18 +39,14 @@ jobs:
           fixtures-metadata.json \
           plugins \
           --extra-db-filename extra_database.db
-        # Package the config with the plugins, excluding test-only plugin secrets
-        # that reference temporary files outside the deployed container.
-        jq 'del(.plugins)' fixtures-config.json > plugins/fixtures-config.json
     - name: Build docs.db
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
+      if: ${{ github.ref == 'refs/heads/main' }}
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
     - name: Set up the alternate-route demo
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         echo '
         from datasette import hookimpl
@@ -84,7 +58,6 @@ jobs:
         ' > plugins/alternative_route.py
         cp fixtures.db fixtures2.db
     - name: And the counters writable stored query demo
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       run: |
         cat > plugins/counters.py <<EOF
         from datasette import hookimpl
@@ -124,15 +97,12 @@ jobs:
     #     cat metadata.json
     - id: auth
       name: Authenticate to Google Cloud
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/auth@v3
       with:
         credentials_json: ${{ secrets.GCP_SA_KEY }}
     - name: Set up Cloud SDK
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       uses: google-github-actions/setup-gcloud@v3
     - name: Deploy to Cloud Run
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' }}
       env:
         LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
       run: |-
@@ -147,16 +117,16 @@ jobs:
             --plugins-dir=plugins \
             --branch=$GITHUB_SHA \
             --version-note=$GITHUB_SHA \
-            --extra-options="--config plugins/fixtures-config.json --setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
+            --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
             --install 'datasette-ephemeral-tables>=0.2.2' \
             --service "datasette-latest$SUFFIX" \
             --secret $LATEST_DATASETTE_SECRET
-    - name: Upload latest documentation database to S3 (only for main)
+    - name: Deploy to docs as well (only for main)
-      if: ${{ steps.deployment-prerequisites.outputs.available == 'true' && github.ref == 'refs/heads/main' }}
+      if: ${{ github.ref == 'refs/heads/main' }}
-      env:
-        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
       run: |-
-        # Keep development documentation separate from the stable release database.
+        # Deploy docs.db to a different service
-        s3-credentials put-object datasette-docs latest/docs.db docs.db \
+        datasette publish cloudrun docs.db \
-          --content-type application/octet-stream
+            --branch=$GITHUB_SHA \
+            --version-note=$GITHUB_SHA \
+            --extra-options="--setting template_debug 1" \
+            --service=datasette-docs-latest

.github/workflows/playwright.yml

@@ -2,15 +2,9 @@ name: Playwright
 
 on:
   push:
-    branches:
-    - main
   pull_request:
   workflow_dispatch:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
 permissions:
   contents: read
 

.github/workflows/prettier.yml

@@ -1,15 +1,6 @@
 name: Check JavaScript for conformance with Prettier
 
-on:
+on: [push]
-  push:
-    branches:
-    - main
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read

.github/workflows/publish.yml

@@ -2,7 +2,7 @@ name: Publish Python Package
 
 on:
   release:
-    types: [published]
+    types: [created]
 
 permissions:
   contents: read
@@ -51,8 +51,6 @@ jobs:
     - name: Publish
       uses: pypa/gh-action-pypi-publish@release/v1
 
-  # After the first non-prerelease 1.0 release, disable this job on 0.65.x,
-  # even for later 0.65 releases, so they cannot overwrite the 1.0 stable docs.
   deploy_static_docs:
     runs-on: ubuntu-latest
     needs: [deploy]
@@ -68,20 +66,26 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install . --group dev
-        python -m pip install sphinx-to-sqlite==0.1a1 "s3-credentials>=0.17"
+        python -m pip install sphinx-to-sqlite==0.1a1
     - name: Build docs.db
       run: |-
         cd docs
         DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
         sphinx-to-sqlite ../docs.db _build
         cd ..
-    - name: Upload stable documentation database to S3
+    - id: auth
-      env:
+      name: Authenticate to Google Cloud
-        AWS_ACCESS_KEY_ID: ${{ secrets.S3_DATASETTE_DOCS_ACCESS_KEY }}
+      uses: google-github-actions/auth@v2
-        AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DATASETTE_DOCS_SECRET_KEY }}
+      with:
+        credentials_json: ${{ secrets.GCP_SA_KEY }}
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v3
+    - name: Deploy stable-docs.datasette.io to Cloud Run
       run: |-
-        s3-credentials put-object datasette-docs docs.db docs.db \
+        gcloud config set run/region us-central1
-          --content-type application/octet-stream
+        gcloud config set project datasette-222320
+        datasette publish cloudrun docs.db \
+            --service=datasette-docs-stable
 
   deploy_docker:
     runs-on: ubuntu-latest

.github/workflows/spellcheck.yml

@@ -1,15 +1,6 @@
 name: Check spelling in documentation
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-    - main
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read

.github/workflows/test-coverage.yml

@@ -0,0 +1,40 @@
+name: Calculate test coverage
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out datasette
+      uses: actions/checkout@v7
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.12'
+        cache: 'pip'
+        cache-dependency-path: '**/pyproject.toml'
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install . --group dev
+        python -m pip install pytest-cov
+    - name: Run tests
+      run: |-
+        ls -lah
+        cat .coveragerc
+        pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x
+        ls -lah
+    - name: Upload coverage report
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: coverage.xml

.github/workflows/test-pyodide.yml

@@ -2,15 +2,9 @@ name: Test in Pyodide with shot-scraper
 
 on:
   push:
-    branches:
-    - main
   pull_request:
   workflow_dispatch:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
 permissions:
   contents: read
 

.github/workflows/test-sqlite-support.yml

@@ -1,15 +1,6 @@
 name: Test SQLite versions
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-    - main
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -21,10 +12,10 @@ jobs:
     strategy:
       matrix:
         platform: [ubuntu-latest]
-        python-version: ["3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         sqlite-version: [
           #"3", # latest version
-          #"3.46",
+          "3.46",
           #"3.45",
           #"3.27",
           #"3.26",

.github/workflows/test.yml

@@ -1,15 +1,6 @@
 name: Test
 
-on:
+on: [push, pull_request]
-  push:
-    branches:
-    - main
-  pull_request:
-  workflow_dispatch:
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 permissions:
   contents: read
@@ -20,20 +11,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
-        include:
-        - python-version: "3.14"
-          coverage: true
     steps:
     - uses: actions/checkout@v7
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v7
+      uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
         allow-prereleases: true
         cache: pip
         cache-dependency-path: pyproject.toml
-        check-latest: true
     - name: Build extension for --load-extension test
       run: |-
         (cd tests && gcc ext.c -fPIC -shared -o ext.so)
@@ -41,27 +28,12 @@ jobs:
       run: |
         pip install . --group dev
         pip freeze
-    - name: Install pytest-cov
-      if: ${{ matrix.coverage }}
-      run: pip install pytest-cov
     - name: Run tests
       run: |
-        if [ "${{ matrix.coverage }}" = "true" ]; then
-          COV="--cov=datasette --cov-config=.coveragerc"
-          pytest -n auto -m "not serial" $COV --cov-report=
-          pytest -m "serial" $COV --cov-append --cov-report xml:coverage.xml --cov-report term
-        else
         pytest -n auto -m "not serial"
         pytest -m "serial"
-        fi
         # And the test that exceeds a localhost HTTPS server
         tests/test_datasette_https_server.sh
-    - name: Upload coverage report
-      if: ${{ matrix.coverage }}
-      uses: codecov/codecov-action@v5
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        files: coverage.xml
     - name: Black
       run: |
         black --version

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11-slim-bookworm AS build
+FROM python:3.11.0-slim-bullseye as build
 
 # Version of Datasette to install, e.g. 0.55
 #   docker build . -t datasette --build-arg VERSION=0.55

Justfile

@@ -49,18 +49,13 @@ export DATASETTE_SECRET := "not_a_secret"
   uv run cog -r README.md docs/*.rst
 
 # Serve live docs on localhost:8000
-@docs: shots cog blacken-docs
+@docs: cog blacken-docs
   uv run make -C docs livehtml
 
 # Build docs as static HTML
 @docs-build: cog blacken-docs
   rm -rf docs/_build && cd docs && uv run make html
 
-# Take any missing documentation screenshots defined in docs/shots.yml
-@shots:
-  uv run --group shots shot-scraper install
-  cd docs && uv run --group shots shot-scraper multi shots.yml --no-clobber --reduced-motion --retina
-
 # Apply Black
 @black:
   uv run black datasette tests

README.md

@@ -36,7 +36,7 @@ You can also install it using `pip` or `pipx`:
 
     pip install datasette
 
-Datasette requires Python 3.10 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
+Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
 
 ## Basic usage
 

datasette/__init__.py

@@ -1,7 +1,6 @@
 from datasette.permissions import Permission  # noqa
 from datasette.version import __version_info__, __version__  # noqa
 from datasette.events import Event  # noqa
-from datasette.background_tasks import BackgroundTask, BackgroundTaskSupervisor  # noqa
 from datasette.tokens import TokenHandler, TokenInvalid, TokenRestrictions  # noqa
 from datasette.utils.asgi import (  # noqa
     Forbidden,

datasette/app.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import contextvars
 from collections.abc import Iterable, Sequence
 from typing import TYPE_CHECKING, Any
 
@@ -27,7 +28,7 @@ import urllib.parse
 from concurrent import futures
 from pathlib import Path
 
-import httpx2
+import httpx
 from itsdangerous import BadSignature, URLSafeSerializer
 from jinja2 import (
     ChoiceLoader,
@@ -41,7 +42,6 @@ from jinja2.exceptions import TemplateNotFound
 from markupsafe import Markup, escape
 
 from . import stored_queries, write_sql
-from .background_tasks import BackgroundTask, BackgroundTaskSupervisor
 from .column_types import SQLiteType
 from .csrf import CrossOriginProtectionMiddleware
 from .database import Database, QueryInterrupted
@@ -49,16 +49,6 @@ from .events import Event
 from .plugins import DEFAULT_PLUGINS, get_plugins, pm
 from .renderer import json_renderer
 from .resources import DatabaseResource, TableResource
-from .telemetry import (
-    TelemetryMiddleware,
-    _in_datasette_client,
-    clamp_http_method,
-    register_datasette,
-    request_span,
-    tracer,
-    unregister_datasette,
-)
-from .telemetry_registry import HTTP_ROUTE, STARTUP
 from .tokens import TokenInvalid
 from .tracer import AsgiTracer
 from .url_builder import Urls
@@ -155,7 +145,6 @@ from .views.stored_queries import (
 )
 from .views.table import (
     TableAutocompleteView,
-    TableCountView,
     TableDropView,
     TableFragmentView,
     TableInsertView,
@@ -175,7 +164,8 @@ app_root = Path(__file__).parent.parent
 logger = logging.getLogger(__name__)
 
 
-# _in_datasette_client is defined in telemetry.py to avoid a circular import
+# Context variable to track when code is executing within a datasette.client request
+_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False)
 
 
 class _DatasetteClientContext:
@@ -325,7 +315,7 @@ def _permission_cache_key(actor, action, parent, child):
     actor_key = (
         json.dumps(actor, sort_keys=True, default=repr) if actor is not None else None
     )
-    return (actor_key, action.name, parent, action.normalize_child(child))
+    return (actor_key, action, parent, child)
 
 
 async def favicon(request, send):
@@ -432,7 +422,6 @@ class Datasette:
         default_deny=False,
     ):
         self._startup_invoked = False
-        self._shutdown_invoked = False
         self._closed = False
         assert config_dir is None or isinstance(
             config_dir, Path
@@ -464,11 +453,8 @@ class Datasette:
         self.databases = collections.OrderedDict()
         self.actions = {}  # .invoke_startup() will populate this
         self._column_types = {}  # .invoke_startup() will populate this
-        self._setup_db_done = False
-        self._suppress_background_tasks = False
         try:
             self._refresh_schemas_lock = asyncio.Lock()
-            self._startup_lock = asyncio.Lock()
         except RuntimeError as rex:
             # Workaround for intermittent test failure, see:
             # https://github.com/simonw/datasette/issues/1802
@@ -476,10 +462,8 @@ class Datasette:
                 loop = asyncio.new_event_loop()
                 asyncio.set_event_loop(loop)
                 self._refresh_schemas_lock = asyncio.Lock()
-                self._startup_lock = asyncio.Lock()
             else:
                 raise
-        self._background_tasks = BackgroundTaskSupervisor(self)
         self.crossdb = crossdb
         self.nolock = nolock
         if memory or crossdb or not self.files:
@@ -651,8 +635,6 @@ class Datasette:
         self.root_enabled = False
         self.default_deny = default_deny
         self.client = DatasetteClient(self)
-        # Last, so metric callbacks never see a partially initialized instance
-        register_datasette(self)
 
     async def apply_metadata_json(self):
         # Apply any metadata entries from metadata.json to the internal tables
@@ -793,8 +775,6 @@ class Datasette:
         # This must be called for Datasette to be in a usable state
         if self._startup_invoked:
             return
-        # Group spans created during startup under a single parent span
-        with tracer.start_as_current_span(STARTUP):
         # Register event classes
         event_classes = []
         for hook in pm.hook.register_events(datasette=self):
@@ -831,9 +811,7 @@ class Datasette:
             if hook:
                 for ct_cls in hook:
                     if ct_cls.name in self._column_types:
-                            raise StartupError(
+                        raise StartupError(f"Duplicate column type name: {ct_cls.name}")
-                                f"Duplicate column type name: {ct_cls.name}"
-                            )
                     self._column_types[ct_cls.name] = ct_cls
 
         for hook in pm.hook.prepare_jinja2_environment(
@@ -980,8 +958,6 @@ class Datasette:
         if self._closed:
             return
         self._closed = True
-        # Stop reporting metrics before closing databases
-        unregister_datasette(self)
         first_exception = None
         dbs = list(self.databases.values()) + [self._internal_database]
         for db in dbs:
@@ -1553,28 +1529,15 @@ class Datasette:
         conn.row_factory = sqlite3.Row
         conn.text_factory = lambda x: str(x, "utf-8", "replace")
         if self.sqlite_extensions and database != INTERNAL_DB_NAME:
-            # Extension loading is only enabled for as long as it takes to
-            # load the configured extensions. Leaving it enabled would let
-            # anyone who can execute SQL call load_extension() themselves.
             conn.enable_load_extension(True)
-            try:
             for extension in self.sqlite_extensions:
                 # "extension" is either a string path to the extension
                 # or a 2-item tuple that specifies which entrypoint to load.
                 if isinstance(extension, tuple):
                     path, entrypoint = extension
-                        if sys.version_info >= (3, 12):
+                    conn.execute("SELECT load_extension(?, ?)", [path, entrypoint])
-                            conn.load_extension(path, entrypoint=entrypoint)
                 else:
-                            # Connection.load_extension() only gained the
+                    conn.execute("SELECT load_extension(?)", [extension])
-                            # entrypoint argument in Python 3.12
-                            conn.execute(
-                                "SELECT load_extension(?, ?)", [path, entrypoint]
-                            )
-                    else:
-                        conn.load_extension(extension)
-            finally:
-                conn.enable_load_extension(False)
         if self.setting("cache_size_kb"):
             conn.execute(f"PRAGMA cache_size=-{self.setting('cache_size_kb')}")
         # pylint: disable=no-member
@@ -1767,145 +1730,8 @@ class Datasette:
         sql, params = await build_allowed_resources_sql(
             self, actor, action, parent=parent, include_is_private=include_is_private
         )
-        if action == "view-table":
-            sql, params = await self._apply_derived_table_permissions_to_sql(
-                sql,
-                params,
-                actor=actor,
-                parent=parent,
-                include_is_private=include_is_private,
-            )
         return ResourcesSQL(sql, params)
 
-    async def _allowed_derived_table_source(
-        self, database, source, *, actor, dependencies
-    ):
-        """Check an immediate source, denying sources that are themselves derived."""
-        if any(
-            TableResource.normalize_child(table)
-            == TableResource.normalize_child(source)
-            for table in dependencies
-        ):
-            return False
-        # The source has no dependency in this map. Evaluate its own permission
-        # and prerequisites without starting another dependency check.
-        verdicts = await self._allowed_many(
-            actions=["view-table"],
-            resource=TableResource(database, source),
-            actor=actor,
-            check_derived=False,
-        )
-        return verdicts["view-table"]
-
-    async def _apply_derived_table_permissions_to_sql(
-        self,
-        sql,
-        params,
-        *,
-        actor,
-        parent,
-        include_is_private,
-    ):
-        databases = (
-            [(parent, self.databases[parent])]
-            if parent in self.databases
-            else ([] if parent is not None else list(self.databases.items()))
-        )
-        dependency_maps = dict(
-            zip(
-                (name for name, _ in databases),
-                await asyncio.gather(
-                    *(db.derived_table_dependencies() for _, db in databases)
-                ),
-            )
-        )
-        dependencies = [
-            (database_name, child, source)
-            for database_name, dependency_map in dependency_maps.items()
-            for child, source in dependency_map.items()
-        ]
-        if not dependencies:
-            return sql, params
-
-        sources = sorted(
-            {(database_name, source) for database_name, _, source in dependencies}
-        )
-        actor_verdicts = await asyncio.gather(
-            *(
-                self._allowed_derived_table_source(
-                    database_name,
-                    source,
-                    actor=actor,
-                    dependencies=dependency_maps[database_name],
-                )
-                for database_name, source in sources
-            )
-        )
-        actor_allowed = dict(zip(sources, actor_verdicts))
-
-        anonymous_allowed = {}
-        if include_is_private:
-            anonymous_verdicts = await asyncio.gather(
-                *(
-                    self._allowed_derived_table_source(
-                        database_name,
-                        source,
-                        actor=None,
-                        dependencies=dependency_maps[database_name],
-                    )
-                    for database_name, source in sources
-                )
-            )
-            anonymous_allowed = dict(zip(sources, anonymous_verdicts))
-
-        wrapped_params = dict(params)
-        derived_rows = [
-            [
-                database_name,
-                child,
-                int(actor_allowed[(database_name, source)]),
-                *(
-                    [int(anonymous_allowed[(database_name, source)])]
-                    if include_is_private
-                    else []
-                ),
-            ]
-            for database_name, child, source in dependencies
-        ]
-        derived_param = "_datasette_derived_permissions"
-        while derived_param in wrapped_params:
-            derived_param += "_"
-        wrapped_params[derived_param] = json.dumps(derived_rows)
-
-        derived_columns = "parent, child, source_allowed"
-        select_columns = "allowed.parent, allowed.child, allowed.reason"
-        if include_is_private:
-            derived_columns += ", source_anonymous_allowed"
-            select_columns += (
-                ", CASE WHEN derived.source_anonymous_allowed = 0 "
-                "THEN 1 ELSE allowed.is_private END AS is_private"
-            )
-        wrapped_sql = f"""
-WITH derived_permissions({derived_columns}) AS (
-  SELECT
-    json_extract(value, '$[0]'),
-    json_extract(value, '$[1]'),
-    json_extract(value, '$[2]')
-    {", json_extract(value, '$[3]')" if include_is_private else ""}
-  FROM json_each(:{derived_param})
-),
-allowed AS (
-{sql}
-)
-SELECT {select_columns}
-FROM allowed
-LEFT JOIN derived_permissions AS derived
-  ON allowed.parent = derived.parent AND allowed.child = derived.child COLLATE NOCASE
-WHERE COALESCE(derived.source_allowed, 1) = 1
-ORDER BY allowed.parent, allowed.child
-""".strip()
-        return wrapped_sql, wrapped_params
-
     async def allowed_resources(
         self,
         action: str,
@@ -2108,12 +1934,6 @@ ORDER BY allowed.parent, allowed.child
             )
             # {"edit-schema": True, "drop-table": True, "insert-row": False}
         """
-        return await self._allowed_many(
-            actions=actions, resource=resource, actor=actor, check_derived=True
-        )
-
-    async def _allowed_many(self, *, actions, resource, actor, check_derived):
-        """Evaluate permissions, optionally applying the one-hop source policy."""
         from datasette.permissions import (
             _permission_check_cache,
             _skip_permission_checks,
@@ -2151,7 +1971,7 @@ ORDER BY allowed.parent, allowed.child
         to_check = []
         for name in expanded:
             if cache is not None:
-                key = _permission_cache_key(actor, self.actions[name], parent, child)
+                key = _permission_cache_key(actor, name, parent, child)
                 if key in cache:
                     final[name] = cache[key]
                     continue
@@ -2167,28 +1987,6 @@ ORDER BY allowed.parent, allowed.child
                 child=child,
             )
 
-        if (
-            check_derived
-            and "view-table" in to_check
-            and raw.get("view-table")
-            and isinstance(resource, TableResource)
-            and parent in self.databases
-        ):
-            dependencies = await self.databases[parent].derived_table_dependencies()
-            source = next(
-                (
-                    source
-                    for table, source in dependencies.items()
-                    if TableResource.normalize_child(table)
-                    == TableResource.normalize_child(child)
-                ),
-                None,
-            )
-            if source is not None:
-                raw["view-table"] = await self._allowed_derived_table_source(
-                    parent, source, actor=actor, dependencies=dependencies
-                )
-
         def resolve(name):
             # final verdict = own rules AND verdict of also_requires chain
             if name in final:
@@ -2206,9 +2004,7 @@ ORDER BY allowed.parent, allowed.child
         # Cache the freshly computed checks
         if cache is not None:
             for name in to_check:
-                cache[
+                cache[_permission_cache_key(actor, name, parent, child)] = final[name]
-                    _permission_cache_key(actor, self.actions[name], parent, child)
-                ] = final[name]
 
         # Log every check (including cache hits) for the debug page,
         # dependencies before the actions that required them
@@ -2299,18 +2095,6 @@ ORDER BY allowed.parent, allowed.child
         from datasette.resources import TableResource
 
         other_table = fk["other_table"]
-        # Foreign key declarations can spell the target with different casing.
-        target_table = (
-            await db.execute(
-                "select name from sqlite_master where type='table' and name=? collate nocase",
-                [other_table],
-            )
-        ).first()
-        if target_table is None:
-            # SQLite accepts a foreign key to a table that does not exist, and
-            # linking to it would only lead to a 404
-            return {}
-        other_table = target_table[0]
         other_column = fk["other_column"]
         if other_column is None:
             other_pks = await db.primary_keys(other_table)
@@ -2494,21 +2278,6 @@ ORDER BY allowed.parent, allowed.child
         )
         return d
 
-    def _tasks(self):
-        return {
-            "tasks": [
-                {
-                    "name": t.name,
-                    "state": t.state,
-                    "function": t.function,
-                    "started_at": t.started_at,
-                    "exception": repr(t.exception) if t.exception else None,
-                }
-                for t in self._background_tasks.tasks()
-            ],
-            "launched": self._background_tasks.launched,
-        }
-
     def _actor(self, request):
         return {"actor": request.actor}
 
@@ -2615,8 +2384,6 @@ ORDER BY allowed.parent, allowed.child
             datasette=self,
         ):
             extra_vars = await await_me_maybe(extra_vars)
-            if extra_vars is None:
-                continue
             assert isinstance(
                 extra_vars, dict
             ), f"extra_vars is of type {type(extra_vars)}"
@@ -2679,7 +2446,7 @@ ORDER BY allowed.parent, allowed.child
     ):
         data = {"a": actor}
         if expire_after:
-            expires_at = int(time.time()) + expire_after
+            expires_at = int(time.time()) + (24 * 60 * 60)
             data["e"] = baseconv.base62.encode(expires_at)
         response.set_cookie("ds_actor", self.sign(data, "actor"))
 
@@ -2799,12 +2566,6 @@ ORDER BY allowed.parent, allowed.child
             ),
             r"/-/threads(\.(?P<format>json))?$",
         )
-        add_route(
-            JsonDataView.as_view(
-                self, "tasks.json", self._tasks, permission="permissions-debug"
-            ),
-            r"/-/tasks(\.(?P<format>json))?$",
-        )
         add_route(
             JsonDataView.as_view(
                 self,
@@ -2979,10 +2740,6 @@ ORDER BY allowed.parent, allowed.child
             TableSetColumnTypeView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/set-column-type$",
         )
-        add_route(
-            TableCountView.as_view(self),
-            r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/count$",
-        )
         add_route(
             TableFragmentView.as_view(self),
             r"/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)/-/fragment$",
@@ -3046,130 +2803,26 @@ ORDER BY allowed.parent, allowed.child
             raise RowNotFound(db.name, table_name, pk_values)
         return ResolvedRow(db, table_name, sql, params, pks, pk_values, results.first())
 
-    async def _startup_sequence(self):
-        """Idempotently run the full startup sequence: table counts for
-        immutable databases, then invoke_startup(). Safe to call more than
-        once and safe to call concurrently - callers block until whichever
-        call got there first has finished.
-
-        This is the single entry point used by both AsgiLifespan (so
-        real deployments finish startup before accepting requests) and
-        AsgiRunOnFirstRequest (the fallback for hosts that never send
-        lifespan events, e.g. DatasetteClient's httpx2.ASGITransport), and
-        `datasette serve` (cli.py) calls it too. The fast path below checks
-        both `_startup_invoked` and `_setup_db_done` - not just the former -
-        so that a bare `await ds.invoke_startup()` made by a caller ahead of
-        `_startup_sequence()` (which only sets `_startup_invoked`) can't
-        make this method skip the immutable-database table-count precompute.
-        """
-        if self._startup_invoked and self._setup_db_done:
-            return
-        async with self._startup_lock:
-            if self._startup_invoked and self._setup_db_done:
-                return
-            if not self._setup_db_done:
-                # First time server starts up, calculate table counts for
-                # immutable databases
-                for database in self.databases.values():
-                    if not database.is_mutable:
-                        await database.table_counts(limit=60 * 60 * 1000)
-                self._setup_db_done = True
-            await self.invoke_startup()
-
-    def add_background_task(self, func, name=None) -> BackgroundTask:
-        """Register a piece of supervised background work, typically from
-        a plugin's ``startup`` hook.
-
-        ``func`` must be a coroutine function taking one positional
-        argument, the ``Datasette`` instance - core calls ``func(self)``.
-        Callable any time after ``__init__``: if background tasks haven't
-        launched yet (the common case - most callers are ``startup`` hooks,
-        which run before launch), this buffers the registration until they
-        do; if they've already launched (e.g. called from a request
-        handler after the server is up), the task starts immediately.
-
-        Returns a :class:`~datasette.background_tasks.BackgroundTask`
-        handle (``.name``, ``.state``, ``.task``, ``.exception``,
-        ``.started_at``, ``.function``, ``.cancel()``).
-
-        ``name`` defaults to ``func.__qualname__``; on a name collision a
-        ``-2``, ``-3``, ... suffix is appended, since names are how
-        ``/-/tasks`` and log messages identify work.
-        """
-        return self._background_tasks.add(func, name=name)
-
-    async def start_background_tasks(self):
-        """Run startup (if it hasn't run yet) and launch every registered
-        background task.
-
-        Public entry point for tests, embedders, and headless CLIs (the
-        ``datasette-rss``-style ``fetch --due`` shape) that want supervised
-        background tasks without running a server - equivalent to what
-        happens automatically via ASGI lifespan / the first-request
-        fallback in a served deployment.
-        """
-        await self.invoke_startup()
-        await self._background_tasks.launch_all()
-
-    async def _launch_background_tasks(self):
-        """Idempotently launch every registered background task. Private:
-        this is the entry point wired into the lifecycle trigger lists
-        (the second entry in both ``AsgiLifespan`` and
-        ``AsgiRunOnFirstRequest``'s ``on_startup``, after
-        ``_startup_sequence``) - not something plugins or embedders should
-        call directly; use ``add_background_task`` /
-        ``start_background_tasks`` instead.
-
-        Positioned after ``_startup_sequence`` in both trigger lists so
-        launch always happens once every plugin's ``startup`` hook has had
-        a chance to register work - the ordering guarantee that makes
-        ``add_background_task`` useful. No-ops when
-        ``_suppress_background_tasks`` is set (the ``--get`` CLI path: its
-        one-shot TestClient request flows through the full ASGI stack,
-        including the first-request fallback, but must never launch
-        long-lived background work).
-        """
-        if self._suppress_background_tasks:
-            return
-        await self._background_tasks.launch_all()
-
-    async def invoke_shutdown(self):
-        """Run the graceful teardown sequence: plugin ``shutdown`` hooks,
-        then cancel and drain supervised background tasks, then close
-        every database.
-        """
-        if self._shutdown_invoked:
-            return
-        self._shutdown_invoked = True
-        for hook in pm.hook.shutdown(datasette=self):
-            try:
-                await await_me_maybe(hook)
-            except Exception:
-                logging.getLogger("datasette").exception("shutdown hook failed")
-        await self._background_tasks.cancel_all(grace=5.0)
-        self.close()
-
     def app(self):
         """Returns an ASGI app function that serves the whole of Datasette"""
         routes = self._routes()
 
+        async def setup_db():
+            # First time server starts up, calculate table counts for immutable databases
+            for database in self.databases.values():
+                if not database.is_mutable:
+                    await database.table_counts(limit=60 * 60 * 1000)
+
+        async def _close_on_shutdown():
+            self.close()
+
         asgi = CrossOriginProtectionMiddleware(DatasetteRouter(self, routes), self)
         if self.setting("trace_debug"):
             asgi = AsgiTracer(asgi)
-        asgi = AsgiLifespan(
+        asgi = AsgiLifespan(asgi, on_shutdown=[_close_on_shutdown])
-            asgi,
+        asgi = AsgiRunOnFirstRequest(asgi, on_startup=[setup_db, self.invoke_startup])
-            on_startup=[self._startup_sequence, self._launch_background_tasks],
-            on_shutdown=[self.invoke_shutdown],
-        )
         for wrapper in pm.hook.asgi_wrapper(datasette=self):
             asgi = wrapper(asgi)
-        asgi = AsgiRunOnFirstRequest(
-            asgi,
-            on_startup=[self._startup_sequence, self._launch_background_tasks],
-        )
-        # Outermost, so spans from plugin middleware and first-request
-        # startup are children of the request span
-        asgi = TelemetryMiddleware(asgi)
         return asgi
 
 
@@ -3207,50 +2860,6 @@ class DatasetteRouter:
             receive,
             max_post_body_bytes=self.ds.setting("max_post_body_bytes"),
         )
-        match, view = resolve_routes(self.routes, path)
-        is_static = view is favicon or getattr(view, "_datasette_static", False)
-        original_send = send
-
-        async def send(message):
-            if message["type"] == "http.response.start" and not (
-                is_static and message["status"] in (200, 304)
-            ):
-                # Decide privacy after rendering, including for streaming responses
-                # and error handlers. A public primary resource can still include
-                # private labels, actor navigation, or cookie-dependent content.
-                headers = list(message.get("headers", []))
-                personalized = (
-                    request.actor is not None
-                    or "cookie" in request.headers
-                    or "authorization" in request.headers
-                    or any(key.lower() == b"set-cookie" for key, _ in headers)
-                )
-                if personalized:
-                    headers = [
-                        (key, value)
-                        for key, value in headers
-                        if key.lower() != b"cache-control"
-                    ]
-                    headers.append((b"cache-control", b"private, no-store"))
-
-                # Anonymous responses must not be reused for credentialed requests.
-                # Preserve any additional variation specified by views or plugins.
-                vary = [
-                    part.strip()
-                    for key, value in headers
-                    if key.lower() == b"vary"
-                    for part in value.split(b",")
-                    if part.strip()
-                ]
-                if b"*" not in vary:
-                    for name in (b"Cookie", b"Authorization"):
-                        if name.lower() not in {part.lower() for part in vary}:
-                            vary.append(name)
-                headers = [(k, v) for k, v in headers if k.lower() != b"vary"]
-                headers.append((b"vary", b", ".join(vary)))
-                message = dict(message, headers=headers)
-            await original_send(message)
-
         # Populate request_messages if ds_messages cookie is present
         try:
             request._messages = self.ds.unsign(
@@ -3290,18 +2899,12 @@ class DatasetteRouter:
             return await self.handle_401(request, send, token_error)
         scope_modifications["actor"] = actor or default_actor
         scope = dict(scope, **scope_modifications)
-        request.scope = scope
+
+        match, view = resolve_routes(self.routes, path)
 
         if match is None:
             return await self.handle_404(request, send)
 
-        # Now the route is known, add it to the request span
-        span = request_span(scope)
-        if span is not None:
-            route = match.re.pattern
-            span.set_attribute(HTTP_ROUTE, route)
-            span.update_name(f"{clamp_http_method(request.method)} {route}")
-
         new_scope = dict(scope, url_route={"kwargs": match.groupdict()})
         request.scope = new_scope
         try:
@@ -3612,14 +3215,14 @@ class DatasetteClient:
         with _DatasetteClientContext():
             if skip_permission_checks:
                 with SkipPermissions():
-                    async with httpx2.AsyncClient(
+                    async with httpx.AsyncClient(
-                        transport=httpx2.ASGITransport(app=self.app),
+                        transport=httpx.ASGITransport(app=self.app),
                         cookies=kwargs.pop("cookies", None),
                     ) as client:
                         return await getattr(client, method)(self._fix(path), **kwargs)
             else:
-                async with httpx2.AsyncClient(
+                async with httpx.AsyncClient(
-                    transport=httpx2.ASGITransport(app=self.app),
+                    transport=httpx.ASGITransport(app=self.app),
                     cookies=kwargs.pop("cookies", None),
                 ) as client:
                     return await getattr(client, method)(self._fix(path), **kwargs)
@@ -3666,10 +3269,10 @@ class DatasetteClient:
             method: HTTP method (e.g., "GET", "POST", "PUT")
             path: The path to request
             skip_permission_checks: If True, bypass all permission checks for this request
-            **kwargs: Additional arguments to pass to httpx2
+            **kwargs: Additional arguments to pass to httpx
 
         Returns:
-            httpx2.Response: The response from the request
+            httpx.Response: The response from the request
         """
         from datasette.permissions import SkipPermissions
 
@@ -3678,16 +3281,16 @@ class DatasetteClient:
         with _DatasetteClientContext():
             if skip_permission_checks:
                 with SkipPermissions():
-                    async with httpx2.AsyncClient(
+                    async with httpx.AsyncClient(
-                        transport=httpx2.ASGITransport(app=self.app),
+                        transport=httpx.ASGITransport(app=self.app),
                         cookies=kwargs.pop("cookies", None),
                     ) as client:
                         return await client.request(
                             method, self._fix(path, avoid_path_rewrites), **kwargs
                         )
             else:
-                async with httpx2.AsyncClient(
+                async with httpx.AsyncClient(
-                    transport=httpx2.ASGITransport(app=self.app),
+                    transport=httpx.ASGITransport(app=self.app),
                     cookies=kwargs.pop("cookies", None),
                 ) as client:
                     return await client.request(

datasette/background_tasks.py

@@ -1,227 +0,0 @@
-"""
-Supervised background-task registration for Datasette core.
-
-Plugins that need long-lived background work (a polling loop, a queue
-consumer, a scheduled job runner) register it with
-``datasette.add_background_task(func, name=None)`` - typically from a
-``startup`` plugin hook - instead of fire-and-forgetting their own
-``asyncio.create_task()``. Core owns:
-
-- **references**: every launched ``asyncio.Task`` is kept alive on a
-  :class:`BackgroundTaskSupervisor`, so it can never be silently garbage
-  collected the way an unreferenced ``create_task()`` call can be;
-- **launch timing**: registered work is buffered until
-  :meth:`BackgroundTaskSupervisor.launch_all` runs, which core arranges to
-  happen only after *every* plugin's ``startup`` hook has finished - so
-  a task that depends on another plugin having registered something first
-  doesn't need ``tryfirst=True`` ordering tricks;
-- **crash surfacing**: an unhandled exception in a background task is
-  logged with its full traceback to the ``datasette.background_tasks``
-  logger and recorded on the handle, instead of becoming an "Task
-  exception was never retrieved" warning nobody sees;
-- **cancellation**: :meth:`BackgroundTaskSupervisor.cancel_all` cancels
-  every task still running and waits (with a grace period) for them to
-  actually stop.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import datetime
-import functools
-import logging
-from collections.abc import Awaitable, Callable
-
-logger = logging.getLogger("datasette.background_tasks")
-
-
-def _utcnow_iso() -> str:
-    return datetime.datetime.now(datetime.timezone.utc).isoformat()
-
-
-def _function_path(func: Callable) -> str:
-    """Describe the callable without guessing which plugin registered it."""
-    while isinstance(func, functools.partial):
-        func = func.func
-    if not hasattr(func, "__qualname__"):
-        func = type(func).__call__
-    return f"{func.__module__}.{func.__qualname__}"
-
-
-class BackgroundTask:
-    """A handle to a single piece of supervised background work.
-
-    States: ``registered`` (added but not yet launched) -> ``running`` ->
-    one of ``completed`` (returned cleanly), ``crashed`` (raised an
-    exception other than ``CancelledError`` - see ``.exception``), or
-    ``cancelled`` (``.cancel()`` was called, or it was still running at
-    shutdown).
-    """
-
-    def __init__(
-        self,
-        name: str,
-        func: Callable[[object], Awaitable[None]],
-    ):
-        self.name = name
-        self.state = "registered"
-        self.task: asyncio.Task | None = None
-        self.exception: BaseException | None = None
-        self.started_at: str | None = None
-        self.function = _function_path(func)
-        self._func = func
-        self._supervisor: BackgroundTaskSupervisor | None = None
-
-    def cancel(self) -> None:
-        """Cancel this task.
-
-        If it has already been launched, cancels the underlying
-        ``asyncio.Task`` - its state becomes ``cancelled`` once the
-        cancellation is observed (asynchronously, via the task's done
-        callback). If it has not been launched yet, this is a no-op as
-        far as asyncio is concerned (there's no task to cancel) but it
-        deregisters the handle from its supervisor so it never runs.
-        """
-        if self.task is not None:
-            self.task.cancel()
-        elif self._supervisor is not None:
-            self._supervisor._deregister(self)
-
-    def __repr__(self) -> str:
-        return f"<BackgroundTask name={self.name!r} state={self.state!r}>"
-
-
-class BackgroundTaskSupervisor:
-    """Owns registration and launch of every :class:`BackgroundTask` for a
-    single ``Datasette`` instance.
-
-    Registration (:meth:`add`) is separate from launch
-    (:meth:`launch_all`): plugins register work whenever convenient
-    (typically from a ``startup`` hook, but request handlers can register
-    dynamic per-job work too), and it either sits buffered until
-    :meth:`launch_all` runs, or - if :meth:`launch_all` has already run -
-    starts immediately.
-
-    Strong references to every :class:`BackgroundTask` (and its
-    ``asyncio.Task``) are kept for the life of the instance, by design -
-    that's what makes the enrichments-style "fire-and-forget task gets
-    garbage collected mid-flight" bug impossible here. There is currently
-    no pruning of completed/crashed/cancelled tasks, so a plugin that
-    dynamically registers many short-lived tasks over a long process
-    lifetime (a per-job registration pattern, e.g. one task per queued
-    job) will grow this list without bound. That's an accepted v1
-    trade-off in favour of full introspection (``/-/tasks``); revisit
-    with a pruning or capping policy if unbounded growth is reported in
-    practice.
-    """
-
-    def __init__(self, datasette):
-        self._datasette = datasette
-        self._tasks: list[BackgroundTask] = []
-        self._names = set()
-        self._launched = False
-        self._lock = asyncio.Lock()
-
-    def add(self, func, name=None) -> BackgroundTask:
-        base_name = name or getattr(func, "__qualname__", None) or repr(func)
-        actual_name = self._unique_name(base_name)
-        handle = BackgroundTask(actual_name, func)
-        handle._supervisor = self
-        self._tasks.append(handle)
-        self._names.add(actual_name)
-        if self._launched:
-            self._launch_one(handle)
-        return handle
-
-    def _unique_name(self, base_name: str) -> str:
-        if base_name not in self._names:
-            return base_name
-        n = 2
-        while f"{base_name}-{n}" in self._names:
-            n += 1
-        return f"{base_name}-{n}"
-
-    def _deregister(self, handle: BackgroundTask) -> None:
-        try:
-            self._tasks.remove(handle)
-        except ValueError:
-            pass
-        self._names.discard(handle.name)
-
-    def _launch_one(self, handle: BackgroundTask) -> None:
-        handle.state = "running"
-        handle.started_at = _utcnow_iso()
-        handle.task = asyncio.create_task(
-            handle._func(self._datasette), name=handle.name
-        )
-        handle.task.add_done_callback(functools.partial(_on_task_done, handle))
-
-    async def launch_all(self) -> None:
-        """Launch every currently-registered task that hasn't launched
-        yet. Idempotent and safe to call concurrently: subsequent (or
-        racing) calls are no-ops once the first has set ``self._launched``.
-        """
-        if self._launched:
-            return
-        async with self._lock:
-            if self._launched:
-                return
-            self._launched = True
-            for handle in list(self._tasks):
-                if handle.task is None:
-                    self._launch_one(handle)
-
-    async def cancel_all(self, grace: float = 5.0) -> None:
-        """Cancel every task that isn't already done, then wait up to
-        ``grace`` seconds for them to actually finish. Stragglers still
-        running after that are logged by name (but left to finish or not
-        on their own - this does not forcibly kill them, asyncio has no
-        mechanism for that).
-        """
-        handles_by_task = {
-            handle.task: handle for handle in self._tasks if handle.task is not None
-        }
-        pending = [task for task in handles_by_task if not task.done()]
-        for task in pending:
-            task.cancel()
-        if not pending:
-            return
-        _done, not_done = await asyncio.wait(pending, timeout=grace)
-        if not_done:
-            names = sorted(handles_by_task[task].name for task in not_done)
-            logger.warning(
-                "%d background task(s) did not finish within the %.1fs grace "
-                "period after cancellation: %s",
-                len(names),
-                grace,
-                ", ".join(names),
-            )
-
-    def tasks(self) -> list[BackgroundTask]:
-        """Return every registered :class:`BackgroundTask`, launched or
-        not, in registration order. Used by the ``/-/tasks`` debug
-        endpoint.
-        """
-        return list(self._tasks)
-
-    @property
-    def launched(self) -> bool:
-        """Whether :meth:`launch_all` has run yet - lets ``/-/tasks``
-        distinguish "no tasks registered" from "tasks registered but
-        nothing has armed the launch yet" without reaching for the
-        private ``_launched`` attribute.
-        """
-        return self._launched
-
-
-def _on_task_done(handle: BackgroundTask, task: asyncio.Task) -> None:
-    if task.cancelled():
-        handle.state = "cancelled"
-        return
-    exc = task.exception()
-    if exc is not None:
-        handle.state = "crashed"
-        handle.exception = exc
-        logger.error("Background task %r crashed", handle.name, exc_info=exc)
-        return
-    handle.state = "completed"

datasette/cli.py

@@ -157,11 +157,7 @@ async def inspect_(files, sqlite_extensions):
     app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
     data = {}
     for name, database in app.databases.items():
-
+        tables = await database.execute_fn(lambda conn: inspect_tables(conn, {}))
-        def _inspect_tables(conn):
-            return inspect_tables(conn, {})
-
-        tables = await database.execute_fn(_inspect_tables)
         data[name] = {
             "hash": database.hash,
             "size": database.size,
@@ -501,7 +497,6 @@ def uninstall(packages, yes):
     "--internal",
     type=click.Path(),
     help="Path to a persistent Datasette internal SQLite database",
-    envvar="DATASETTE_INTERNAL",
 )
 def serve(
     files,
@@ -668,6 +663,16 @@ def serve(
         # Private utility mechanism for writing unit tests
         return ds
 
+    # Run async soundness checks before startup hooks, since invoke_startup
+    # now populates internal tables which requires querying each database
+    run_sync(lambda: check_databases(ds))
+
+    # Run the "startup" plugin hooks
+    try:
+        run_sync(ds.invoke_startup)
+    except StartupError as e:
+        raise click.ClickException(e.args[0])
+
     if headers and not get:
         raise click.ClickException("--headers can only be used with --get")
 
@@ -675,19 +680,6 @@ def serve(
         raise click.ClickException("--token can only be used with --get")
 
     if get:
-        # --get means we don't run Uvicorn at all
-        run_sync(lambda: check_databases(ds))
-
-        try:
-            run_sync(ds.invoke_startup)
-        except StartupError as e:
-            raise click.ClickException(e.args[0])
-
-        # --get never launches background tasks: TestClient's request below
-        # flows through the full ASGI stack, including the
-        # AsgiRunOnFirstRequest fallback, which would otherwise launch them.
-        ds._suppress_background_tasks = True
-
         client = TestClient(ds)
         request_headers = {}
         if token:
@@ -712,23 +704,6 @@ def serve(
         sys.exit(exit_code)
         return
 
-    # check_databases, invoke_startup() and the uvicorn server all run on a
-    # single event loop, so that anything a plugin's "startup" hook schedules
-    # on the loop (asyncio.create_task, Lock/Queue/Event objects, ...) is
-    # still alive when the server starts handling requests.
-    async def _serve_async():
-        # Populate internal catalog tables before invoke_startup
-        await check_databases(ds)
-
-        # Run the full startup sequence (immutable-database table-count
-        # precompute + the "startup" plugin hooks) via the same entry point
-        # AsgiLifespan/AsgiRunOnFirstRequest use, so it's not skipped when
-        # uvicorn's lifespan.startup fires moments later.
-        try:
-            await ds._startup_sequence()
-        except StartupError as e:
-            raise click.ClickException(e.args[0])
-
     # Start the server
     url = None
     if root:
@@ -740,7 +715,7 @@ def serve(
     if open_browser:
         if url is None:
             # Figure out most convenient URL - to table, database or homepage
-                path = await initial_path_for_datasette(ds)
+            path = run_sync(lambda: initial_path_for_datasette(ds))
             url = f"http://{host}:{port}{path}"
         webbrowser.open(url)
     uvicorn_kwargs = {
@@ -756,10 +731,7 @@ def serve(
         uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile
     if ssl_certfile:
         uvicorn_kwargs["ssl_certfile"] = ssl_certfile
-        server = uvicorn.Server(uvicorn.Config(ds.app(), **uvicorn_kwargs))
+    uvicorn.run(ds.app(), **uvicorn_kwargs)
-        await server.serve()
-
-    asyncio.run(_serve_async())
 
 
 @cli.command()

datasette/database.py

@@ -1,54 +1,18 @@
 import asyncio
 import atexit
-import contextvars
 import inspect
 import os
 import queue
 import sys
 import tempfile
 import threading
-import time
 import uuid
 from collections import namedtuple
 from pathlib import Path
 
 import sqlite_utils
-from opentelemetry import context as otel_context_api
-from opentelemetry.trace import Status, StatusCode
 
 from .inspect import inspect_hash
-from .telemetry import (
-    callback_name,
-    linked_root_span_kwargs,
-    record_operation_duration,
-    record_query_interrupted,
-    record_write_queue_wait,
-    sql_attribute,
-    sql_operation_name,
-    tracer,
-)
-from .telemetry_registry import (
-    CALLBACK,
-    DB_NAMESPACE,
-    DB_OPERATION_NAME,
-    DB_QUERY,
-    DB_QUERY_EXECUTE,
-    DB_QUERY_TEXT,
-    DB_SYSTEM,
-    DB_WRITE_EXECUTE,
-    DB_WRITE_QUEUE_WAIT,
-    EXECUTEMANY,
-    EXECUTESCRIPT,
-    INTERRUPTED,
-    ISOLATED_CONNECTION,
-    PARAM_COUNT,
-    PARAM_SETS,
-    ROWS_RETURNED,
-    SQL_ERROR_SUPPRESSED,
-    TIME_LIMIT_MS,
-    TRANSACTION,
-    TRUNCATED,
-)
 from .tracer import trace
 from .utils import (
     call_with_supported_arguments,
@@ -65,7 +29,7 @@ from .utils import (
     table_columns,
 )
 from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables
-from .utils.sqlite import sqlite_derived_table_dependencies, sqlite_hidden_table_names
+from .utils.sqlite import sqlite_hidden_table_names
 
 connections = threading.local()
 
@@ -121,7 +85,6 @@ class Database:
         self.cached_hash = None
         self.cached_size = None
         self._cached_table_counts = None
-        self._cached_derived_table_dependencies = None
         self._write_thread = None
         self._write_queue = None
         self._closed = False
@@ -130,9 +93,8 @@ class Database:
         # These are used when in non-threaded mode:
         self._read_connection = None
         self._write_connection = None
-        # Track file and memory connections, including reads on worker threads,
+        # This is used to track all file connections so they can be closed
-        # so close() can release all of them from the calling thread.
+        self._all_file_connections = []
-        self._all_connections = []
         if not is_temp_disk:
             self.mode = mode
 
@@ -183,12 +145,9 @@ class Database:
             )
             if not write:
                 conn.execute("PRAGMA query_only=1")
-            self._all_connections.append(conn)
             return conn
         if self.is_memory:
-            conn = sqlite3.connect(":memory:", uri=True, check_same_thread=False)
+            return sqlite3.connect(":memory:", uri=True)
-            self._all_connections.append(conn)
-            return conn
 
         # mode=ro or immutable=1?
         if self.is_mutable:
@@ -205,7 +164,7 @@ class Database:
         conn = sqlite3.connect(
             f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs
         )
-        self._all_connections.append(conn)
+        self._all_file_connections.append(conn)
         if self.is_temp_disk and not self._wal_enabled:
             conn.execute("PRAGMA journal_mode=WAL")
             self._wal_enabled = True
@@ -242,13 +201,13 @@ class Database:
             except Exception:  # noqa: BLE001, S110
                 # Shutdown teardown - a failed pending write must not block close()
                 pass
-        # Close anything still tracked in _all_connections
+        # Close anything still tracked in _all_file_connections
-        for connection in self._all_connections:
+        for connection in self._all_file_connections:
             try:
                 connection.close()
             except Exception:  # noqa: BLE001, S110
                 pass
-        self._all_connections = []
+        self._all_file_connections = []
         # Drop per-thread cached read connections we can reach
         try:
             delattr(connections, self._thread_local_id)
@@ -287,43 +246,19 @@ class Database:
         return_all=False,
         returning_limit=EXECUTE_WRITE_RETURNING_LIMIT,
         transaction=True,
-        time_limit_ms=2000,
     ):
         self._check_not_closed()
         if returning_limit < 0:
             raise ValueError("returning_limit must be >= 0")
 
-        def execute_sql(conn):
+        def _inner(conn):
             cursor = conn.execute(sql, params or [])
             return ExecuteWriteResult.from_cursor(
                 cursor, return_all=return_all, returning_limit=returning_limit
             )
 
-        def _inner(conn):
+        with trace("sql", database=self.name, sql=sql.strip(), params=params):
-            try:
+            results = await self.execute_write_fn(
-                if time_limit_ms is None:
-                    return execute_sql(conn)
-                with sqlite_timelimit(conn, time_limit_ms):
-                    return execute_sql(conn)
-            except (sqlite3.OperationalError, sqlite3.DatabaseError) as e:
-                if e.args == ("interrupted",):
-                    raise QueryInterrupted(e, sql, params)
-                raise
-
-        with trace(  # noqa: SIM117
-            "sql", database=self.name, sql=sql.strip(), params=params
-        ):
-            with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
-                span.set_attribute(DB_SYSTEM, "sqlite")
-                span.set_attribute(DB_NAMESPACE, self.name)
-                span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
-                operation_name = sql_operation_name(sql)
-                if operation_name:
-                    span.set_attribute(DB_OPERATION_NAME, operation_name)
-                if params:
-                    span.set_attribute(PARAM_COUNT, len(params))
-                with record_operation_duration(self.name, "write"):
-                    results = await self._execute_write_fn(
                 _inner, block=block, request=request, transaction=transaction
             )
         return results
@@ -334,17 +269,8 @@ class Database:
         def _inner(conn):
             return conn.executescript(sql)
 
-        with trace(  # noqa: SIM117
+        with trace("sql", database=self.name, sql=sql.strip(), executescript=True):
-            "sql", database=self.name, sql=sql.strip(), executescript=True
+            results = await self.execute_write_fn(
-        ):
-            # No db.operation.name, since the script can contain multiple statements
-            with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
-                span.set_attribute(DB_SYSTEM, "sqlite")
-                span.set_attribute(DB_NAMESPACE, self.name)
-                span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
-                span.set_attribute(EXECUTESCRIPT, True)
-                with record_operation_duration(self.name, "write"):
-                    results = await self._execute_write_fn(
                 _inner, block=block, transaction=False, request=request
             )
         return results
@@ -366,19 +292,9 @@ class Database:
         with trace(
             "sql", database=self.name, sql=sql.strip(), executemany=True
         ) as kwargs:
-            with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
+            results, count = await self.execute_write_fn(
-                span.set_attribute(DB_SYSTEM, "sqlite")
-                span.set_attribute(DB_NAMESPACE, self.name)
-                span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
-                span.set_attribute(EXECUTEMANY, True)
-                operation_name = sql_operation_name(sql)
-                if operation_name:
-                    span.set_attribute(DB_OPERATION_NAME, operation_name)
-                with record_operation_duration(self.name, "write"):
-                    results, count = await self._execute_write_fn(
                 _inner, block=block, request=request
             )
-                span.set_attribute(PARAM_SETS, count)
             kwargs["count"] = count
         return results
 
@@ -395,27 +311,19 @@ class Database:
             finally:
                 isolated_connection.close()
                 try:
-                    self._all_connections.remove(isolated_connection)
+                    self._all_file_connections.remove(isolated_connection)
                 except ValueError:
-                    # May already have been cleared by close().
+                    # Was probably a memory connection
                     pass
 
-        with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
-            span.set_attribute(DB_SYSTEM, "sqlite")
-            span.set_attribute(DB_NAMESPACE, self.name)
-            span.set_attribute(CALLBACK, callback_name(fn))
-            # Immutable databases run this on the read pool, not the write queue
-            with record_operation_duration(self.name, "write" if write else "read"):
         if self.ds.executor is None:
             # non-threaded mode
             return _run()
         if not write:
-                    # Immutable database - no writes can ever occur, so there
+            # Immutable database - no writes can ever occur, so there is no
-                    # is no write queue to block; run against a fresh
+            # write queue to block; run against a fresh read-only connection
-                    # read-only connection
-                    ctx = contextvars.copy_context()
             return await asyncio.get_running_loop().run_in_executor(
-                        self.ds.executor, ctx.run, _run
+                self.ds.executor, _run
             )
         # Threaded mode - send to write thread
         return await self._send_to_write_thread(fn, isolated_connection=True)
@@ -423,30 +331,11 @@ class Database:
     async def analyze_sql(self, sql, params=None) -> SQLAnalysis:
         self._check_not_closed()
 
-        def _analyze_sql(conn):
+        return await self.execute_isolated_fn(
-            return analyze_sql_tables(conn, sql, params, database_name=self.name)
+            lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name)
-
-        return await self.execute_isolated_fn(_analyze_sql)
-
-    async def execute_write_fn(self, fn, block=True, transaction=True, request=None):
-        """Run `fn(conn)` on the write connection, traced as a `db.query` span.
-
-        The SQL-string write methods call `_execute_write_fn()` directly to
-        avoid creating a second span.
-        """
-        self._check_not_closed()
-        # Record the name before _wrap_fn_with_hooks() wraps fn
-        name = callback_name(fn)
-        with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
-            span.set_attribute(DB_SYSTEM, "sqlite")
-            span.set_attribute(DB_NAMESPACE, self.name)
-            span.set_attribute(CALLBACK, name)
-            with record_operation_duration(self.name, "write"):
-                return await self._execute_write_fn(
-                    fn, block=block, transaction=transaction, request=request
         )
 
-    async def _execute_write_fn(self, fn, block=True, transaction=True, request=None):
+    async def execute_write_fn(self, fn, block=True, transaction=True, request=None):
         self._check_not_closed()
         pending_events = []
 
@@ -465,15 +354,6 @@ class Database:
                     result = fn(self._write_connection)
             else:
                 result = fn(self._write_connection)
-            if not block:
-                # There is no write thread here, so the write has already
-                # finished. Hand back the same (task_id, reply_future) shape
-                # _send_to_write_thread() returns, with the future already
-                # resolved, so the block=False path below is identical in
-                # both modes.
-                reply_future = asyncio.get_running_loop().create_future()
-                reply_future.set_result(result)
-                result = (uuid.uuid4(), reply_future)
         else:
             result = await self._send_to_write_thread(
                 fn, block=block, transaction=transaction
@@ -545,22 +425,11 @@ class Database:
             )
             self._write_thread.name = f"_execute_writes for database {self.name}"
             self._write_thread.start()
-        task_id = uuid.uuid4()
+        task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
         loop = asyncio.get_running_loop()
         reply_future = loop.create_future()
-        # Capture the OpenTelemetry context and enqueue time for the write thread
         self._write_queue.put(
-            WriteTask(
+            WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction)
-                fn,
-                task_id,
-                loop,
-                reply_future,
-                isolated_connection,
-                transaction,
-                otel_context_api.get_current(),
-                time.time_ns(),
-                block,
-            )
         )
         if block:
             return await reply_future
@@ -574,8 +443,6 @@ class Database:
         conn = None
         try:
             conn = self.connect(write=True)
-            # Threads do not inherit the caller's context, so any spans
-            # created by prepare_connection hooks here are root spans
             self.ds._prepare_connection(conn, self.name)
         except Exception as e:  # noqa: BLE001
             # Stored and re-raised to whoever queues the next write
@@ -590,49 +457,21 @@ class Database:
                         # Best-effort close as the write thread exits
                         pass
                 return
-            # block=True: the caller awaits the result, so the write spans
-            # are children of the caller's span. The token must be detached
-            # in the finally block or the context leaks into later writes.
-            # block=False: the caller may finish first, so the write spans
-            # are root spans with a link back to the caller's span.
-            token = None
-            write_span_kwargs = {}
-            if task.block:
-                token = otel_context_api.attach(task.otel_context)
-            else:
-                write_span_kwargs = linked_root_span_kwargs(task.otel_context)
-            try:
             exception = None
             result = None
-                # Span covers the time from enqueue to dequeue
-                dequeued_at_ns = time.time_ns()
-                tracer.start_span(
-                    DB_WRITE_QUEUE_WAIT,
-                    start_time=task.enqueued_at_ns,
-                    **write_span_kwargs,
-                ).end(end_time=dequeued_at_ns)
-                record_write_queue_wait(self.name, dequeued_at_ns - task.enqueued_at_ns)
             if conn_exception is not None:
                 exception = conn_exception
             elif task.isolated_connection:
                 try:
-                        with tracer.start_as_current_span(
-                            DB_WRITE_EXECUTE, **write_span_kwargs
-                        ) as span:
-                            span.set_attribute(
-                                ISOLATED_CONNECTION,
-                                task.isolated_connection,
-                            )
-                            span.set_attribute(TRANSACTION, task.transaction)
                     isolated_connection = self.connect(write=True)
                     try:
                         result = task.fn(isolated_connection)
                     finally:
                         isolated_connection.close()
                         try:
-                                    self._all_connections.remove(isolated_connection)
+                            self._all_file_connections.remove(isolated_connection)
                         except ValueError:
-                                    # May already have been cleared by close().
+                            # Was probably a memory connection
                             pass
                 except Exception as e:  # noqa: BLE001
                     # Write thread must survive any task failure or the database wedges
@@ -641,14 +480,6 @@ class Database:
                     exception = e
             else:
                 try:
-                        with tracer.start_as_current_span(
-                            DB_WRITE_EXECUTE, **write_span_kwargs
-                        ) as span:
-                            span.set_attribute(
-                                ISOLATED_CONNECTION,
-                                task.isolated_connection,
-                            )
-                            span.set_attribute(TRANSACTION, task.transaction)
                     if task.transaction:
                         with conn:
                             conn.execute("BEGIN IMMEDIATE")
@@ -660,31 +491,8 @@ class Database:
                     sys.stderr.flush()
                     exception = e
             _deliver_write_result(task, result, exception)
-            finally:
-                if token is not None:
-                    otel_context_api.detach(token)
 
     async def execute_fn(self, fn):
-        """Run `fn(conn)` on a read connection, traced as a `db.query` span.
-
-        `execute()` calls `_execute_fn()` directly to avoid creating a second
-        span.
-        """
-        self._check_not_closed()
-
-        def fn_in_execute_span(conn):
-            # Runs on the worker thread
-            with tracer.start_as_current_span(DB_QUERY_EXECUTE):
-                return fn(conn)
-
-        with tracer.start_as_current_span(DB_QUERY, kind=DB_QUERY.kind) as span:
-            span.set_attribute(DB_SYSTEM, "sqlite")
-            span.set_attribute(DB_NAMESPACE, self.name)
-            span.set_attribute(CALLBACK, callback_name(fn))
-            with record_operation_duration(self.name, "read"):
-                return await self._execute_fn(fn_in_execute_span)
-
-    async def _execute_fn(self, fn):
         self._check_not_closed()
         if self.ds.executor is None:
             # non-threaded mode
@@ -704,11 +512,7 @@ class Database:
 
         with self._pending_execute_futures_lock:
             self._check_not_closed()
-            # Run in a copy of the caller's context so spans created in the
+            future = self.ds.executor.submit(in_thread)
-            # thread have the correct parent. This needs a fresh copy for
-            # each submit, since a Context cannot be entered concurrently.
-            ctx = contextvars.copy_context()
-            future = self.ds.executor.submit(ctx.run, in_thread)
             self._pending_execute_futures.add(future)
         future.add_done_callback(self._remove_pending_execute_future)
         return await asyncio.wrap_future(future)
@@ -725,22 +529,12 @@ class Database:
         """Executes sql against db_name in a thread"""
         self._check_not_closed()
         page_size = page_size or self.ds.page_size
-        time_limit_ms = self.ds.sql_time_limit_ms
-        # Callers that pass a shorter custom_time_limit, such as table counts
-        # and facet suggestions, expect timeouts, so they are not span errors
-        timeout_expected = bool(custom_time_limit) and custom_time_limit < time_limit_ms
-        if timeout_expected:
-            time_limit_ms = custom_time_limit
 
         def sql_operation_in_thread(conn):
-            # Expected timeouts and errors with log_sql_errors=False are not
+            time_limit_ms = self.ds.sql_time_limit_ms
-            # recorded as span errors, so exceptions are handled explicitly
+            if custom_time_limit and custom_time_limit < time_limit_ms:
-            with tracer.start_as_current_span(
+                time_limit_ms = custom_time_limit
-                DB_QUERY_EXECUTE,
+
-                record_exception=False,
-                set_status_on_exception=False,
-            ) as execute_span:
-                try:
             with sqlite_timelimit(conn, time_limit_ms):
                 try:
                     cursor = conn.cursor()
@@ -764,16 +558,6 @@ class Database:
                         )
                         sys.stderr.flush()
                     raise
-                except QueryInterrupted as e:
-                    if not timeout_expected:
-                        execute_span.record_exception(e)
-                        execute_span.set_status(Status(StatusCode.ERROR, str(e)))
-                    raise
-                except Exception as e:
-                    if log_sql_errors:
-                        execute_span.record_exception(e)
-                        execute_span.set_status(Status(StatusCode.ERROR, str(e)))
-                    raise
 
             if truncate:
                 return Results(rows, truncated, cursor.description)
@@ -781,45 +565,8 @@ class Database:
             else:
                 return Results(rows, False, cursor.description)
 
-        with trace(  # noqa: SIM117
+        with trace("sql", database=self.name, sql=sql.strip(), params=params):
-            "sql", database=self.name, sql=sql.strip(), params=params
+            results = await self.execute_fn(sql_operation_in_thread)
-        ):
-            with tracer.start_as_current_span(
-                DB_QUERY,
-                kind=DB_QUERY.kind,
-                record_exception=False,
-                set_status_on_exception=False,
-            ) as span:
-                span.set_attribute(DB_SYSTEM, "sqlite")
-                span.set_attribute(DB_NAMESPACE, self.name)
-                span.set_attribute(DB_QUERY_TEXT, sql_attribute(sql))
-                span.set_attribute(TIME_LIMIT_MS, time_limit_ms)
-                operation_name = sql_operation_name(sql)
-                if operation_name:
-                    span.set_attribute(DB_OPERATION_NAME, operation_name)
-                if params:
-                    span.set_attribute(PARAM_COUNT, len(params))
-                try:
-                    with record_operation_duration(self.name, "read"):
-                        results = await self._execute_fn(sql_operation_in_thread)
-                except QueryInterrupted as e:
-                    span.set_attribute(INTERRUPTED, True)
-                    if not timeout_expected:
-                        span.set_status(Status(StatusCode.ERROR, str(e)))
-                        span.record_exception(e)
-                        record_query_interrupted(self.name)
-                    raise
-                except Exception as e:
-                    # log_sql_errors=False callers, such as facet suggestion,
-                    # expect some queries to fail
-                    if log_sql_errors:
-                        span.record_exception(e)
-                        span.set_status(Status(StatusCode.ERROR, str(e)))
-                    else:
-                        span.set_attribute(SQL_ERROR_SUPPRESSED, True)
-                    raise
-                span.set_attribute(TRUNCATED, results.truncated)
-                span.set_attribute(ROWS_RETURNED, len(results.rows))
         return results
 
     @property
@@ -910,32 +657,17 @@ class Database:
         )
         return [r[0] for r in results.rows]
 
-    # Named functions rather than lambdas give more useful datasette.callback
-    # span attributes
-
     async def table_columns(self, table):
-        def _table_columns(conn):
+        return await self.execute_fn(lambda conn: table_columns(conn, table))
-            return table_columns(conn, table)
-
-        return await self.execute_fn(_table_columns)
 
     async def table_column_details(self, table):
-        def _table_column_details(conn):
+        return await self.execute_fn(lambda conn: table_column_details(conn, table))
-            return table_column_details(conn, table)
-
-        return await self.execute_fn(_table_column_details)
 
     async def primary_keys(self, table):
-        def _primary_keys(conn):
+        return await self.execute_fn(lambda conn: detect_primary_keys(conn, table))
-            return detect_primary_keys(conn, table)
-
-        return await self.execute_fn(_primary_keys)
 
     async def fts_table(self, table):
-        def _fts_table(conn):
+        return await self.execute_fn(lambda conn: detect_fts(conn, table))
-            return detect_fts(conn, table)
-
-        return await self.execute_fn(_fts_table)
 
     async def label_column_for_table(self, table):
         explicit_label_column = (await self.ds.table_config(self.name, table)).get(
@@ -1027,17 +759,6 @@ class Database:
 
         return hidden_tables
 
-    async def derived_table_dependencies(self):
-        """Return implementation tables and the tables they derive from."""
-        schema_version = (await self.execute("PRAGMA schema_version")).first()[0]
-        if (
-            self._cached_derived_table_dependencies is None
-            or self._cached_derived_table_dependencies[0] != schema_version
-        ):
-            dependencies = await self.execute_fn(sqlite_derived_table_dependencies)
-            self._cached_derived_table_dependencies = (schema_version, dependencies)
-        return self._cached_derived_table_dependencies[1]
-
     async def view_names(self):
         results = await self.execute("select name from sqlite_master where type='view'")
         return [r[0] for r in results.rows]
@@ -1133,28 +854,16 @@ def _apply_write_wrapper(fn, wrapper_factory, track_event):
 
 class WriteTask:
     __slots__ = (
-        "block",
-        "enqueued_at_ns",
         "fn",
         "isolated_connection",
         "loop",
-        "otel_context",
         "reply_future",
         "task_id",
         "transaction",
     )
 
     def __init__(
-        self,
+        self, fn, task_id, loop, reply_future, isolated_connection, transaction
-        fn,
-        task_id,
-        loop,
-        reply_future,
-        isolated_connection,
-        transaction,
-        otel_context,
-        enqueued_at_ns,
-        block,
     ):
         self.fn = fn
         self.task_id = task_id
@@ -1162,9 +871,6 @@ class WriteTask:
         self.reply_future = reply_future
         self.isolated_connection = isolated_connection
         self.transaction = transaction
-        self.otel_context = otel_context
-        self.enqueued_at_ns = enqueued_at_ns
-        self.block = block
 
 
 def _deliver_write_result(task, result, exception):

datasette/default_column_types.py

@@ -6,17 +6,6 @@ import markupsafe
 from datasette import hookimpl
 from datasette.column_types import ColumnType, SQLiteType
 
-_HTTP_URL_RE = re.compile(r"https?://\S+", re.IGNORECASE)
-
-
-def _normalize_http_url(value):
-    if not isinstance(value, str):
-        return None
-    normalized = value.strip()
-    if not _HTTP_URL_RE.fullmatch(normalized):
-        return None
-    return normalized
-
 
 class UrlColumnType(ColumnType):
     name = "url"
@@ -26,10 +15,7 @@ class UrlColumnType(ColumnType):
     async def render_cell(self, value, column, table, database, datasette, request):
         if not value or not isinstance(value, str):
             return None
-        normalized = _normalize_http_url(value)
+        escaped = markupsafe.escape(value.strip())
-        if normalized is None:
-            return markupsafe.escape(value.strip())
-        escaped = markupsafe.escape(normalized)
         return markupsafe.Markup(f'<a href="{escaped}">{escaped}</a>')
 
     async def validate(self, value, datasette):
@@ -37,7 +23,7 @@ class UrlColumnType(ColumnType):
             return None
         if not isinstance(value, str):
             return "URL must be a string"
-        if _normalize_http_url(value) is None:
+        if not re.match(r"^https?://\S+$", value.strip()):
             return "Invalid URL"
         return None
 

datasette/default_permissions/config.py

@@ -92,13 +92,6 @@ class ConfigPermissionProcessor:
             # Tables implicitly reference their parent databases
             self.restricted_databases.update(db for db, _ in self.restricted_tables)
 
-        # Resolve identity keys once per action, rather than scanning the
-        # restriction allowlist for every configured table's allow block.
-        self.restricted_table_keys = {
-            (db, self.action_obj.normalize_child(table) if self.action_obj else table)
-            for db, table in self.restricted_tables
-        }
-
     def evaluate_allow_block(self, allow_block: Any) -> bool | None:
         """Evaluate an allow block against the current actor."""
         if allow_block is None:
@@ -132,10 +125,8 @@ class ConfigPermissionProcessor:
         if parent:
             table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {})
             if child:
-                child_key = (
+                table_actions = table_restrictions.get(child, [])
-                    self.action_obj.normalize_child(child) if self.action_obj else child
+                if self.action_checks.intersection(table_actions):
-                )
-                if (parent, child_key) in self.restricted_table_keys:
                     return True
             else:
                 # Parent query should proceed if any child in this database is allowlisted

datasette/default_permissions/restrictions.py

@@ -185,12 +185,8 @@ def restrictions_allow_action(
     # Check table/resource level
     if resource is not None and not isinstance(resource, str) and len(resource) == 2:
         database, table = resource
-        action_obj = datasette.actions.get(action)
+        table_allowed = restrictions.get("r", {}).get(database, {}).get(table)
-        normalize = action_obj.normalize_child if action_obj else lambda name: name
+        if table_allowed is not None:
-        for table_name, table_allowed in (
-            restrictions.get("r", {}).get(database, {}).items()
-        ):
-            if normalize(table_name) == normalize(table):
             assert isinstance(table_allowed, list)
             if to_check.intersection(table_allowed):
                 return True

datasette/default_permissions/sqlite_statistics.py

@@ -1,25 +0,0 @@
-"""Default table-access policy for SQLite optimizer statistics."""
-
-import json
-
-from datasette import hookimpl
-from datasette.permissions import PermissionSQL
-
-
-@hookimpl
-def permission_resources_sql(action):
-    if action != "view-table":
-        return None
-    return PermissionSQL(
-        sql="""
-            SELECT database_name AS parent, value AS child, 0 AS allow,
-                'SQLite statistics tables are denied by default' AS reason
-            FROM catalog_databases
-            CROSS JOIN json_each(:sqlite_statistics_names)
-        """,
-        params={
-            "sqlite_statistics_names": json.dumps(
-                ["sqlite_stat1", "sqlite_stat2", "sqlite_stat3", "sqlite_stat4"]
-            )
-        },
-    )

datasette/facets.py

@@ -39,7 +39,7 @@ def load_facet_configs(request, table_config):
         )
     qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True)
     for key, values in qs_pairs.items():
-        if key == "_facet" or key.startswith("_facet_"):
+        if key.startswith("_facet"):
             # Figure out the facet type
             if key == "_facet":
                 type = "column"
@@ -264,15 +264,10 @@ class ColumnFacet(Facet):
                     column_qs = column
                     if column.startswith("_"):
                         column_qs = f"{column}__exact"
-                    selected_args = {
+                    selected = (column_qs, str(row["value"])) in qs_pairs
-                        key: str(row["value"])
-                        for key in (column_qs, f"{column}__exact")
-                        if (key, str(row["value"])) in qs_pairs
-                    }
-                    selected = bool(selected_args)
                     if selected:
                         toggle_path = path_with_removed_args(
-                            self.request, selected_args
+                            self.request, {column_qs: str(row["value"])}
                         )
                     else:
                         toggle_path = path_with_added_args(

datasette/filters.py

@@ -1,9 +1,8 @@
 import json
-import math
 from typing import ClassVar
 
 from datasette import hookimpl
-from datasette.resources import DatabaseResource, TableResource
+from datasette.resources import DatabaseResource
 from datasette.utils.asgi import BadRequest
 from datasette.views.base import DatasetteError
 
@@ -52,20 +51,13 @@ def search_filters(request, database, table, datasette):
         human_descriptions = []
         extra_context = {}
 
-        # Figure out which trusted fts_table to use. Query string parameters can
+        # Figure out which fts_table to use
-        # repeat this mapping (for backwards compatibility), but must not select
-        # a different table or primary key.
         table_metadata = await datasette.table_config(database, table)
         db = datasette.get_database(database)
-        fts_table = table_metadata.get("fts_table")
+        fts_table = request.args.get("_fts_table")
+        fts_table = fts_table or table_metadata.get("fts_table")
         fts_table = fts_table or await db.fts_table(table)
-        fts_pk = table_metadata.get("fts_pk", "rowid")
+        fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid"))
-        requested_fts_table = request.args.get("_fts_table")
-        requested_fts_pk = request.args.get("_fts_pk")
-        if (requested_fts_table and requested_fts_table != fts_table) or (
-            requested_fts_pk and requested_fts_pk != fts_pk
-        ):
-            raise BadRequest("Invalid _fts_table or _fts_pk")
         search_args = {
             key: request.args[key]
             for key in request.args
@@ -83,11 +75,6 @@ def search_filters(request, database, table, datasette):
         extra_context["supports_search"] = bool(fts_table)
 
         if fts_table and search_args:
-            await datasette.ensure_permission(
-                action="view-table",
-                resource=TableResource(database=database, table=fts_table),
-                actor=request.actor,
-            )
             if "_search" in search_args:
                 # Simple ?_search=xxx
                 search = search_args["_search"]
@@ -148,11 +135,6 @@ def through_filters(request, database, table, datasette):
                 through_table = through_data["table"]
                 other_column = through_data["column"]
                 value = through_data["value"]
-                await datasette.ensure_permission(
-                    action="view-table",
-                    resource=TableResource(database=database, table=through_table),
-                    actor=request.actor,
-                )
                 db = datasette.get_database(database)
                 outgoing_foreign_keys = await db.foreign_keys_for_table(through_table)
                 fk_to_us = next(
@@ -203,17 +185,6 @@ class Filter:
         raise NotImplementedError
 
 
-def _coerce_numeric_filter_value(value):
-    try:
-        return int(value)
-    except ValueError:
-        try:
-            converted = float(value)
-        except ValueError:
-            return value
-        return converted if math.isfinite(converted) else value
-
-
 class TemplatedFilter(Filter):
     def __init__(
         self,
@@ -235,8 +206,8 @@ class TemplatedFilter(Filter):
 
     def where_clause(self, table, column, value, param_counter):
         converted = self.format.format(value)
-        if self.numeric:
+        if self.numeric and converted.isdigit():
-            converted = _coerce_numeric_filter_value(converted)
+            converted = int(converted)
         if self.no_argument:
             kwargs = {"c": _quote_sqlite_identifier(column)}
             converted = None

datasette/handle_exception.py

@@ -59,10 +59,6 @@ def handle_exception(datasette, request, exception):
             body = dict(info)
             body.update(error_body(plain_message or message, status))
             return Response.json(body, status=status, headers=headers)
-        if request.path.split("?")[0].endswith(".csv"):
-            return Response.text(
-                plain_message or message, status=status, headers=headers
-            )
         info.update(
             {
                 "ok": False,

datasette/hookspecs.py

@@ -9,11 +9,6 @@ def startup(datasette):
     """Fires directly after Datasette first starts running"""
 
 
-@hookspec
-def shutdown(datasette):
-    """Called once when the Datasette server is shutting down"""
-
-
 @hookspec
 def asgi_wrapper(datasette):
     """Returns an ASGI middleware callable to wrap our ASGI application with"""
@@ -50,7 +45,7 @@ def extra_body_script(
 def extra_template_vars(
     template, database, table, columns, view_name, request, datasette
 ):
-    """Extra template variables to be made available to the template - can return dict, None, callable or awaitable"""
+    """Extra template variables to be made available to the template - can return dict or callable or awaitable"""
 
 
 @hookspec

datasette/permissions.py

@@ -3,10 +3,6 @@ from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from typing import Any, NamedTuple
 
-_SQLITE_IDENTIFIER_CASE = str.maketrans(
-    "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
-)
-
 # Context variable to track when permission checks should be skipped
 _skip_permission_checks = contextvars.ContextVar(
     "skip_permission_checks", default=False
@@ -53,15 +49,6 @@ class Resource(ABC):
     # Class-level metadata (subclasses must define these)
     name: str = None  # e.g., "table", "database", "model"
     parent_class: type["Resource"] | None = None  # e.g., DatabaseResource for tables
-    case_insensitive_child: bool = False
-
-    @classmethod
-    def normalize_child(cls, child: str | None) -> str | None:
-        """Return a comparison key without changing the resource's display name."""
-        if cls.case_insensitive_child and child is not None:
-            # Match SQLite NOCASE: fold ASCII only, not Unicode lower/casefold.
-            return child.translate(_SQLITE_IDENTIFIER_CASE)
-        return child
 
     # Instance-level optional extra attributes
     reasons: list[str] | None = None
@@ -159,11 +146,6 @@ class Action:
     resource_class: type[Resource] | None = None
     also_requires: str | None = None  # Optional action name that must also be allowed
 
-    def normalize_child(self, child: str | None) -> str | None:
-        if self.resource_class is None:
-            return child
-        return self.resource_class.normalize_child(child)
-
     @property
     def takes_parent(self) -> bool:
         """

datasette/plugins.py

@@ -18,7 +18,6 @@ DEFAULT_PLUGINS = (
     "datasette.actor_auth_cookie",
     "datasette.default_permissions",
     "datasette.default_permissions.tokens",
-    "datasette.default_permissions.sqlite_statistics",
     "datasette.default_actions",
     "datasette.default_column_types",
     "datasette.default_magic_parameters",

datasette/resources.py

@@ -25,7 +25,6 @@ class TableResource(Resource):
 
     name = "table"
     parent_class = DatabaseResource
-    case_insensitive_child = True
 
     def __init__(self, database: str, table: str):
         super().__init__(parent=database, child=table)

datasette/static/column-chooser.js

@@ -1,9 +1,7 @@
-let columnChooserInstanceCounter = 0;
-
 class ColumnChooser extends HTMLElement {
   constructor() {
     super();
-    this.titleId = `column-chooser-title-${++columnChooserInstanceCounter}`;
+    this.attachShadow({ mode: "open" });
 
     // State
     this._items = [];
@@ -28,60 +26,375 @@ class ColumnChooser extends HTMLElement {
     // Bound handlers
     this._onMove = this._onMove.bind(this);
     this._onUp = this._onUp.bind(this);
+
+    this.shadowRoot.innerHTML = `
+      <style>
+        :host {
+          --ink: #0f0f0f;
+          --paper: #eef6ff;
+          --muted: #6b6b6b;
+          --rule: #d8e6f5;
+          --accent: #1a56db;
+          --accent-light: #e8effd;
+          --card: #ffffff;
         }
 
-  connectedCallback() {
+        * { box-sizing: border-box; margin: 0; padding: 0; }
-    if (this._modal) return;
+
-    this.innerHTML = `
+        dialog {
-      <datasette-modal><dialog aria-labelledby="${this.titleId}">
+          border: none;
+          border-radius: var(--modal-border-radius, 0.75rem);
+          padding: 0;
+          margin: auto;
+          width: 100%;
+          max-width: 420px;
+          max-height: min(640px, calc(100vh - 32px));
+          box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
+          animation: slideIn var(--modal-animation-duration, 0.2s) ease-out;
+          overflow: hidden;
+          font-family: system-ui, -apple-system, sans-serif;
+          background: var(--card);
+          -webkit-user-select: none;
+          -webkit-touch-callout: none;
+          -webkit-tap-highlight-color: transparent;
+        }
+
+        dialog[open] {
+          display: flex;
+          flex-direction: column;
+          height: min(640px, calc(100vh - 32px));
+        }
+
+        dialog::backdrop {
+          background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
+          backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+          -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+          animation: fadeIn var(--modal-animation-duration, 0.2s) ease-out;
+        }
+
+        @keyframes slideIn {
+          from {
+            opacity: 0;
+            transform: translateY(-20px) scale(0.95);
+          }
+          to {
+            opacity: 1;
+            transform: translateY(0) scale(1);
+          }
+        }
+
+        @keyframes fadeIn {
+          from { opacity: 0; }
+          to { opacity: 1; }
+        }
+
+        .modal-header {
+          padding: 20px 24px 16px;
+          border-bottom: 1px solid var(--rule);
+          display: flex;
+          align-items: center;
+          justify-content: space-between;
+          flex-shrink: 0;
+        }
+
+        .modal-title {
+          font-size: 1rem;
+          font-weight: 600;
+        }
+
+        .modal-meta {
+          font-family: ui-monospace, monospace;
+          font-size: 0.7rem;
+          color: var(--muted);
+          background: var(--paper);
+          padding: 3px 9px;
+          border-radius: 20px;
+        }
+
+        .list-toolbar {
+          padding: 6px 24px;
+          border-bottom: 1px solid var(--rule);
+          display: flex;
+          gap: 12px;
+          flex-shrink: 0;
+        }
+
+        .list-toolbar button {
+          background: var(--accent-light);
+          border: 1px solid var(--rule);
+          border-radius: 4px;
+          font-family: inherit;
+          font-size: 0.75rem;
+          color: var(--accent);
+          cursor: pointer;
+          padding: 3px 10px;
+          transition: background 0.12s, color 0.12s;
+        }
+        .list-toolbar button:hover { background: var(--accent); color: white; }
+
+        .list-wrap {
+          flex: 1;
+          overflow-y: auto;
+          overflow-x: hidden;
+          position: relative;
+          overscroll-behavior: contain;
+          -webkit-overflow-scrolling: touch;
+        }
+
+        .list-wrap::before,
+        .list-wrap::after {
+          content: '';
+          position: sticky;
+          display: block;
+          left: 0; right: 0;
+          height: 20px;
+          pointer-events: none;
+          z-index: 5;
+          transition: opacity 0.2s;
+        }
+        .list-wrap::before {
+          top: 0;
+          background: linear-gradient(to bottom, rgba(255,255,255,0.9), transparent);
+        }
+        .list-wrap::after {
+          bottom: 0;
+          background: linear-gradient(to top, rgba(255,255,255,0.9), transparent);
+          margin-top: -20px;
+        }
+
+        .scroll-zone {
+          position: absolute;
+          left: 0; right: 0;
+          height: 72px;
+          pointer-events: none;
+          z-index: 10;
+        }
+        .scroll-zone-top { top: 0; }
+        .scroll-zone-bot { bottom: 0; }
+
+        .drag-list {
+          list-style: none;
+          padding: 4px 0;
+        }
+
+        .drag-item {
+          display: flex;
+          align-items: center;
+          background: white;
+          border-bottom: 1px solid var(--rule);
+          user-select: none;
+          -webkit-user-select: none;
+          -webkit-touch-callout: none;
+          position: relative;
+          transition: background 0.08s;
+        }
+
+        .drag-item:last-child { border-bottom: none; }
+
+        .drag-handle {
+          display: flex;
+          align-items: center;
+          justify-content: center;
+          width: 48px;
+          height: 48px;
+          flex-shrink: 0;
+          cursor: grab;
+          color: #c8c4bc;
+          touch-action: none;
+          transition: color 0.15s;
+        }
+
+        .drag-handle:hover { color: var(--accent); }
+        .drag-handle svg { pointer-events: none; display: block; }
+
+        .drag-item-content {
+          display: flex;
+          align-items: center;
+          flex: 1;
+          min-width: 0;
+          cursor: pointer;
+        }
+
+        .drag-item-check {
+          display: flex;
+          align-items: center;
+          width: 32px;
+          height: 48px;
+          flex-shrink: 0;
+        }
+
+        .drag-item-check input[type="checkbox"] {
+          width: 16px;
+          height: 16px;
+          accent-color: var(--accent);
+          cursor: pointer;
+        }
+
+        .drag-item-label {
+          flex: 1;
+          font-size: 0.9rem;
+          line-height: 48px;
+          padding-right: 16px;
+          white-space: nowrap;
+          overflow: hidden;
+          text-overflow: ellipsis;
+          cursor: default;
+        }
+
+        .drag-item.is-dragging {
+          opacity: 0;
+        }
+
+        .drop-indicator {
+          position: absolute;
+          left: 48px;
+          right: 0;
+          height: 2px;
+          background: var(--accent);
+          border-radius: 99px;
+          pointer-events: none;
+          z-index: 20;
+          display: none;
+        }
+        .drop-indicator.top { top: -1px; display: block; }
+        .drop-indicator.bottom { bottom: -1px; display: block; }
+
+        .drag-ghost {
+          position: fixed;
+          pointer-events: none;
+          z-index: 9999;
+          background: white;
+          border-radius: 6px;
+          box-shadow: 0 8px 32px rgba(0,0,0,0.18), 0 2px 8px rgba(0,0,0,0.1);
+          display: flex;
+          align-items: center;
+          border: 1.5px solid var(--accent-light);
+          opacity: 0.97;
+          will-change: transform;
+          font-family: system-ui, -apple-system, sans-serif;
+        }
+
+        .scroll-pulse {
+          position: absolute;
+          left: 50%;
+          transform: translateX(-50%);
+          width: 32px;
+          height: 32px;
+          border-radius: 50%;
+          background: var(--accent);
+          opacity: 0;
+          pointer-events: none;
+          z-index: 10;
+          transition: opacity 0.15s;
+        }
+        .scroll-pulse.top { top: 8px; }
+        .scroll-pulse.bot { bottom: 8px; }
+        .scroll-pulse.active {
+          opacity: 0.18;
+          animation: pulse 0.8s ease-in-out infinite;
+        }
+
+        @keyframes pulse {
+          0%, 100% { transform: translateX(-50%) scale(1); opacity: 0.18; }
+          50% { transform: translateX(-50%) scale(1.5); opacity: 0.07; }
+        }
+
+        .modal-footer {
+          padding: 14px 20px;
+          border-top: 1px solid var(--rule);
+          display: flex;
+          align-items: center;
+          gap: 10px;
+          flex-shrink: 0;
+          background: var(--paper);
+        }
+
+        .footer-info {
+          flex: 1;
+          font-family: ui-monospace, monospace;
+          font-size: 0.68rem;
+          color: var(--muted);
+        }
+
+        .btn {
+          border: none;
+          border-radius: 5px;
+          padding: 9px 20px;
+          font-size: 0.85rem;
+          font-weight: 500;
+          cursor: pointer;
+          touch-action: manipulation;
+          font-family: inherit;
+          transition: background 0.12s;
+        }
+
+        .btn-primary {
+          background: var(--accent);
+          color: white;
+        }
+        .btn-primary:hover { background: #1448c0; }
+
+        .btn-ghost {
+          background: transparent;
+          color: var(--muted);
+          border: 1px solid var(--rule);
+        }
+        .btn-ghost:hover { background: var(--rule); color: var(--ink); }
+
+        .list-wrap::-webkit-scrollbar { width: 5px; }
+        .list-wrap::-webkit-scrollbar-track { background: transparent; }
+        .list-wrap::-webkit-scrollbar-thumb { background: var(--rule); border-radius: 99px; }
+
+        input, textarea { -webkit-user-select: auto; user-select: auto; }
+      </style>
+
+      <dialog aria-labelledby="modalTitle">
           <div class="modal-header">
-            <span class="modal-title" id="${this.titleId}">Choose columns</span>
+            <span class="modal-title" id="modalTitle">Choose columns</span>
-            <span class="modal-meta"></span>
+            <span class="modal-meta" id="selectedCount"></span>
           </div>
           <div class="list-toolbar">
-            <button class="select-all">Select all</button>
+            <button id="selectAllBtn">Select all</button>
-            <button class="deselect-all">Deselect all</button>
+            <button id="deselectAllBtn">Deselect all</button>
           </div>
-          <div class="modal-body list-wrap">
+          <div class="list-wrap" id="listWrap">
-            <div class="scroll-pulse top"></div>
+            <div class="scroll-pulse top" id="pulseTop"></div>
-            <div class="scroll-pulse bot"></div>
+            <div class="scroll-pulse bot" id="pulseBot"></div>
-            <ul class="drag-list"></ul>
+            <ul class="drag-list" id="dragList"></ul>
           </div>
           <div class="modal-footer">
-            <span class="footer-info"></span>
+            <span class="footer-info" id="footerInfo"></span>
-            <button class="modal-btn modal-btn-ghost">Cancel</button>
+            <button class="btn btn-ghost" id="cancelBtn">Cancel</button>
-            <button class="modal-btn modal-btn-primary">Apply</button>
+            <button class="btn btn-primary" id="applyBtn">Apply</button>
           </div>
-      </dialog></datasette-modal>
+      </dialog>
     `;
 
     // DOM refs
-    this._modal = this.querySelector("datasette-modal");
+    this._dialog = this.shadowRoot.querySelector("dialog");
-    this._listWrap = this.querySelector(".list-wrap");
+    this._listWrap = this.shadowRoot.getElementById("listWrap");
-    this._dragList = this.querySelector(".drag-list");
+    this._dragList = this.shadowRoot.getElementById("dragList");
-    this._pulseTop = this.querySelector(".scroll-pulse.top");
+    this._pulseTop = this.shadowRoot.getElementById("pulseTop");
-    this._pulseBot = this.querySelector(".scroll-pulse.bot");
+    this._pulseBot = this.shadowRoot.getElementById("pulseBot");
-    this._selectAllBtn = this.querySelector(".select-all");
+    this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn");
-    this._deselectAllBtn = this.querySelector(".deselect-all");
+    this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn");
-    this._cancelBtn = this.querySelector(".modal-btn-ghost");
+    this._cancelBtn = this.shadowRoot.getElementById("cancelBtn");
-    this._applyBtn = this.querySelector(".modal-btn-primary");
+    this._applyBtn = this.shadowRoot.getElementById("applyBtn");
-    this._countEl = this.querySelector(".modal-meta");
+    this._countEl = this.shadowRoot.getElementById("selectedCount");
-    this._footerEl = this.querySelector(".footer-info");
+    this._footerEl = this.shadowRoot.getElementById("footerInfo");
 
     // Event listeners
     this._selectAllBtn.addEventListener("click", () => this._selectAll());
     this._deselectAllBtn.addEventListener("click", () => this._deselectAll());
-    this._cancelBtn.addEventListener("click", () =>
+    this._cancelBtn.addEventListener("click", () => this._close());
-      this._modal.requestClose("cancel"),
-    );
     this._applyBtn.addEventListener("click", () => this._apply());
-    this._modal.beforeClose = () => {
+    this._dialog.addEventListener("click", (e) => {
-      this._items = this._savedItems ? [...this._savedItems] : this._items;
+      if (e.target === this._dialog) this._close();
-      this._checked = this._savedChecked
+    });
-        ? new Set(this._savedChecked)
+    this._dialog.addEventListener("cancel", (e) => {
-        : this._checked;
+      e.preventDefault();
-      return true;
+      this._close();
-    };
+    });
   }
 
   /**
@@ -101,11 +414,19 @@ class ColumnChooser extends HTMLElement {
     this._savedChecked = new Set(this._checked);
 
     this._render();
-    this._modal.show();
+    this._dialog.showModal();
   }
 
   // ── Internal methods ──
 
+  _close() {
+    this._items = this._savedItems ? [...this._savedItems] : this._items;
+    this._checked = this._savedChecked
+      ? new Set(this._savedChecked)
+      : this._checked;
+    this._dialog.close();
+  }
+
   _selectAll() {
     this._items.forEach((col) => this._checked.add(col));
     this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => {
@@ -124,7 +445,7 @@ class ColumnChooser extends HTMLElement {
 
   _apply() {
     const selected = this._items.filter((col) => this._checked.has(col));
-    this._modal.close();
+    this._dialog.close();
     if (this._onApply) {
       this._onApply(selected);
     }
@@ -151,13 +472,11 @@ class ColumnChooser extends HTMLElement {
           <span class="drag-item-check">
             <input type="checkbox" ${this._checked.has(col) ? "checked" : ""}>
           </span>
-          <span class="drag-item-label"></span>
+          <span class="drag-item-label">${col}</span>
         </label>
         <div class="drop-indicator"></div>
       `;
 
-      li.querySelector(".drag-item-label").textContent = col;
-
       li.querySelector("input").addEventListener("change", (e) => {
         e.target.checked ? this._checked.add(col) : this._checked.delete(col);
         this._updateCounts();
@@ -190,7 +509,7 @@ class ColumnChooser extends HTMLElement {
     this._ghostOffX = e.clientX - rect.left;
     this._ghostOffY = e.clientY - rect.top;
 
-    // Keep the drag preview inside the dialog so it stays above the backdrop.
+    // Build ghost inside shadow DOM
     this._ghost = document.createElement("div");
     this._ghost.className = "drag-ghost";
     this._ghost.style.width = rect.width + "px";
@@ -199,7 +518,7 @@ class ColumnChooser extends HTMLElement {
     this._ghost.querySelector(".drop-indicator")?.remove();
     const h = this._ghost.querySelector(".drag-handle");
     if (h) h.style.color = "var(--accent)";
-    this._modal.dialog.appendChild(this._ghost);
+    this.shadowRoot.appendChild(this._ghost);
 
     srcEl.classList.add("is-dragging");
     this._positionGhost(e.clientX, e.clientY);

datasette/static/edit-tools.js

@@ -915,7 +915,6 @@ function showTableCreateDialogError(state, message) {
 
 function setTableCreateDialogSaving(state, isSaving) {
   state.isSaving = isSaving;
-  state.modal.busy = isSaving;
   state.columnList
     .querySelectorAll("input, select, button")
     .forEach(function (control) {
@@ -2044,7 +2043,8 @@ async function createTableFromDataPreview(state) {
     var tableUrl =
       responseData.table_url ||
       fallbackTableUrl(responseData.table || payload.table);
-    state.modal.close({ restoreFocus: false });
+    state.shouldRestoreFocus = false;
+    state.dialog.close();
     if (tableUrl) {
       location.href = tableUrl;
     } else {
@@ -2118,7 +2118,8 @@ async function saveTableCreateDialog(state) {
     var tableUrl =
       responseData.table_url ||
       fallbackTableUrl(responseData.table || payload.table);
-    state.modal.close({ restoreFocus: false });
+    state.shouldRestoreFocus = false;
+    state.dialog.close();
     if (tableUrl) {
       location.href = tableUrl;
     } else {
@@ -2140,6 +2141,18 @@ function confirmDiscardTableCreateChanges(state) {
   return window.confirm("Discard this new table?");
 }
 
+function closeTableCreateDialogIfConfirmed(state) {
+  if (!state || state.isSaving) {
+    return false;
+  }
+  if (!confirmDiscardTableCreateChanges(state)) {
+    return false;
+  }
+  state.shouldRestoreFocus = true;
+  state.dialog.close();
+  return true;
+}
+
 function ensureTableCreateDialog(manager) {
   if (tableCreateDialogState) {
     return tableCreateDialogState;
@@ -2148,8 +2161,7 @@ function ensureTableCreateDialog(manager) {
     return null;
   }
 
-  var modal = DatasetteModal.create();
+  var dialog = document.createElement("dialog");
-  var dialog = modal.dialog;
   dialog.id = TABLE_CREATE_DIALOG_ID;
   dialog.className = "table-create-dialog";
   dialog.setAttribute("aria-labelledby", "table-create-title");
@@ -2159,7 +2171,7 @@ function ensureTableCreateDialog(manager) {
     </div>
     <form class="table-create-form" method="post" novalidate>
       <p class="table-create-error" id="table-create-error" role="alert" tabindex="-1" hidden></p>
-      <div class="modal-body table-create-fields">
+      <div class="table-create-fields">
         <div class="table-create-field">
           <label class="table-create-label" for="table-create-name">Table name</label>
           <input class="table-create-input table-create-table-name" id="table-create-name" type="text" name="table" required autocomplete="off">
@@ -2186,15 +2198,14 @@ function ensureTableCreateDialog(manager) {
       <div class="modal-footer">
         <a href="#" class="table-create-mode-link table-create-from-data">Create table from data</a>
         <a href="#" class="table-create-mode-link table-create-manual" hidden>Create table manually</a>
-        <button type="button" class="modal-btn modal-btn-ghost table-create-cancel">Cancel</button>
+        <button type="button" class="btn btn-ghost table-create-cancel">Cancel</button>
-        <button type="submit" class="modal-btn modal-btn-primary table-create-save">Create table</button>
+        <button type="submit" class="btn btn-primary table-create-save">Create table</button>
       </div>
     </form>
   `;
-  document.body.appendChild(modal);
+  document.body.appendChild(dialog);
 
   tableCreateDialogState = {
-    modal: modal,
     dialog: dialog,
     form: dialog.querySelector(".table-create-form"),
     title: dialog.querySelector(".modal-title"),
@@ -2214,6 +2225,8 @@ function ensureTableCreateDialog(manager) {
     manualCreateLink: dialog.querySelector(".table-create-manual"),
     cancelButton: dialog.querySelector(".table-create-cancel"),
     saveButton: dialog.querySelector(".table-create-save"),
+    currentButton: null,
+    shouldRestoreFocus: true,
     isSaving: false,
     mode: "manual",
     dataPreviewRows: null,
@@ -2253,7 +2266,7 @@ function ensureTableCreateDialog(manager) {
       tableCreateDialogState.dataTextarea.focus();
       return;
     }
-    modal.requestClose("cancel");
+    closeTableCreateDialogIfConfirmed(tableCreateDialogState);
   });
 
   tableCreateDialogState.createFromDataLink.addEventListener(
@@ -2351,14 +2364,36 @@ function ensureTableCreateDialog(manager) {
     updateTableCreateDialogButtons(tableCreateDialogState);
   });
 
-  modal.beforeClose = function (source) {
+  dialog.addEventListener("click", function (ev) {
-    return confirmDiscardTableCreateChanges(tableCreateDialogState);
+    if (ev.target === dialog) {
-  };
+      closeTableCreateDialogIfConfirmed(tableCreateDialogState);
+    }
+  });
+
+  dialog.addEventListener("keydown", function (ev) {
+    if (ev.key !== "Escape") {
+      return;
+    }
+    ev.preventDefault();
+    closeTableCreateDialogIfConfirmed(tableCreateDialogState);
+  });
+
+  dialog.addEventListener("cancel", function (ev) {
+    ev.preventDefault();
+    closeTableCreateDialogIfConfirmed(tableCreateDialogState);
+  });
 
   dialog.addEventListener("close", function () {
     var state = tableCreateDialogState;
     clearTableCreateDialogError(state);
     setTableCreateDialogSaving(state, false);
+    if (
+      state.shouldRestoreFocus &&
+      state.currentButton &&
+      document.contains(state.currentButton)
+    ) {
+      state.currentButton.focus();
+    }
   });
 
   return tableCreateDialogState;
@@ -2379,12 +2414,15 @@ function openTableCreateDialog(button, manager) {
     menu.open = false;
   }
   state.manager = manager;
-
+  state.currentButton = button;
+  state.shouldRestoreFocus = true;
   state.title.textContent = "Create a table in " + data.databaseName;
   clearTableCreateDialogError(state);
   resetTableCreateDialog(state);
   loadTableCreateForeignKeyTargets(state);
-  state.modal.show({ returnFocusTo: button });
+  if (!state.dialog.open) {
+    state.dialog.showModal();
+  }
   state.tableName.focus();
 }
 
@@ -2410,7 +2448,6 @@ function initTableCreateActions(manager) {
 
 function setRowDeleteDialogBusy(state, isBusy) {
   state.isBusy = isBusy;
-  state.modal.busy = isBusy;
   state.confirmButton.disabled = isBusy;
   state.cancelButton.disabled = isBusy;
   state.confirmButton.textContent = isBusy ? "Deleting..." : "Delete row";
@@ -2657,7 +2694,6 @@ function showTableAlterDialogError(state, message) {
 
 function setTableAlterDialogSaving(state, isSaving) {
   state.isSaving = isSaving;
-  state.modal.busy = isSaving;
   state.cancelButton.disabled = isSaving;
   state.addColumnButton.disabled = isSaving;
   state.backButton.disabled = isSaving;
@@ -3793,7 +3829,8 @@ async function applyTableAlterChanges(state, result) {
       result.columnTypeAssignments || [],
       tableUrl,
     );
-    state.modal.close({ restoreFocus: false });
+    state.shouldRestoreFocus = false;
+    state.dialog.close();
     if (tableAlterResultRenamesTable(result) && tableUrl) {
       window.location.href = tableUrl;
     } else {
@@ -3854,7 +3891,8 @@ async function dropTableFromAlterDialog(state) {
     if (!response.ok || (responseData && responseData.ok === false)) {
       throw rowMutationRequestError(response, responseData);
     }
-    state.modal.close({ restoreFocus: false });
+    state.shouldRestoreFocus = false;
+    state.dialog.close();
     window.location.href = tableAlterDatabaseUrl() || "/";
   } catch (error) {
     setTableAlterDialogSaving(state, false);
@@ -3890,6 +3928,27 @@ function confirmDiscardTableAlterChanges(state) {
   return window.confirm("Discard table changes?");
 }
 
+function closeTableAlterDialogIfConfirmed(state) {
+  if (!state || state.isSaving) {
+    return false;
+  }
+  if (!confirmDiscardTableAlterChanges(state)) {
+    return false;
+  }
+  state.shouldRestoreFocus = true;
+  state.dialog.close();
+  return true;
+}
+
+function closeTableAlterDialog(state) {
+  if (!state || state.isSaving) {
+    return false;
+  }
+  state.shouldRestoreFocus = true;
+  state.dialog.close();
+  return true;
+}
+
 function ensureTableAlterDialog(manager) {
   if (tableAlterDialogState) {
     return tableAlterDialogState;
@@ -3898,8 +3957,7 @@ function ensureTableAlterDialog(manager) {
     return null;
   }
 
-  var modal = DatasetteModal.create();
+  var dialog = document.createElement("dialog");
-  var dialog = modal.dialog;
   dialog.id = TABLE_ALTER_DIALOG_ID;
   dialog.className = "table-alter-dialog";
   dialog.setAttribute("aria-labelledby", "table-alter-title");
@@ -3909,7 +3967,7 @@ function ensureTableAlterDialog(manager) {
     </div>
     <form class="table-alter-form" method="post" novalidate>
       <p class="table-alter-error" id="table-alter-error" role="alert" tabindex="-1" hidden></p>
-      <div class="modal-body table-alter-fields">
+      <div class="table-alter-fields">
         <div class="table-alter-columns">
           <div class="table-alter-column-headings" aria-hidden="true">
             <span>Column</span>
@@ -3928,19 +3986,18 @@ function ensureTableAlterDialog(manager) {
           </div>
         </details>
       </div>
-      <div class="modal-body table-alter-review" hidden></div>
+      <div class="table-alter-review" hidden></div>
       <div class="modal-footer">
-        <button type="button" class="modal-btn modal-btn-danger table-alter-drop" hidden>Drop table</button>
+        <button type="button" class="btn btn-danger table-alter-drop" hidden>Drop table</button>
-        <button type="button" class="modal-btn modal-btn-ghost table-alter-back" hidden>Back</button>
+        <button type="button" class="btn btn-ghost table-alter-back" hidden>Back</button>
-        <button type="button" class="modal-btn modal-btn-ghost table-alter-cancel">Cancel</button>
+        <button type="button" class="btn btn-ghost table-alter-cancel">Cancel</button>
-        <button type="submit" class="modal-btn modal-btn-primary table-alter-save">Review changes</button>
+        <button type="submit" class="btn btn-primary table-alter-save">Review changes</button>
       </div>
     </form>
   `;
-  document.body.appendChild(modal);
+  document.body.appendChild(dialog);
 
   tableAlterDialogState = {
-    modal: modal,
     dialog: dialog,
     form: dialog.querySelector(".table-alter-form"),
     title: dialog.querySelector(".modal-title"),
@@ -3955,6 +4012,8 @@ function ensureTableAlterDialog(manager) {
     dropButton: dialog.querySelector(".table-alter-drop"),
     cancelButton: dialog.querySelector(".table-alter-cancel"),
     saveButton: dialog.querySelector(".table-alter-save"),
+    currentButton: null,
+    shouldRestoreFocus: true,
     isSaving: false,
     initialSignature: "",
     originalTableName: "",
@@ -3996,7 +4055,7 @@ function ensureTableAlterDialog(manager) {
   });
 
   tableAlterDialogState.cancelButton.addEventListener("click", function () {
-    modal.requestClose("cancel");
+    closeTableAlterDialog(tableAlterDialogState);
   });
 
   tableAlterDialogState.dropButton.addEventListener("click", function () {
@@ -4017,17 +4076,36 @@ function ensureTableAlterDialog(manager) {
     }
   });
 
-  modal.beforeClose = function (source) {
+  dialog.addEventListener("click", function (ev) {
-    return (
+    if (ev.target === dialog) {
-      source === "cancel" ||
+      closeTableAlterDialogIfConfirmed(tableAlterDialogState);
-      confirmDiscardTableAlterChanges(tableAlterDialogState)
+    }
-    );
+  });
-  };
+
+  dialog.addEventListener("keydown", function (ev) {
+    if (ev.key !== "Escape") {
+      return;
+    }
+    ev.preventDefault();
+    closeTableAlterDialogIfConfirmed(tableAlterDialogState);
+  });
+
+  dialog.addEventListener("cancel", function (ev) {
+    ev.preventDefault();
+    closeTableAlterDialogIfConfirmed(tableAlterDialogState);
+  });
 
   dialog.addEventListener("close", function () {
     var state = tableAlterDialogState;
     clearTableAlterDialogError(state);
     setTableAlterDialogSaving(state, false);
+    if (
+      state.shouldRestoreFocus &&
+      state.currentButton &&
+      document.contains(state.currentButton)
+    ) {
+      state.currentButton.focus();
+    }
   });
 
   return tableAlterDialogState;
@@ -4048,7 +4126,8 @@ function openTableAlterDialog(button, manager) {
     menu.open = false;
   }
   state.manager = manager;
-
+  state.currentButton = button;
+  state.shouldRestoreFocus = true;
   state.title.textContent = "Alter table " + data.tableName;
   clearTableAlterDialogError(state);
   resetTableAlterDialog(state, data);
@@ -4058,7 +4137,9 @@ function openTableAlterDialog(button, manager) {
     tableAlterForeignKeyTargetsUrl(),
     { filterByType: false },
   );
-  state.modal.show({ returnFocusTo: button });
+  if (!state.dialog.open) {
+    state.dialog.showModal();
+  }
   var firstName = state.columnList.querySelector(".table-alter-column-name");
   if (firstName) {
     firstName.focus();
@@ -4361,8 +4442,7 @@ function ensureRowDeleteDialog(manager) {
     return null;
   }
 
-  var modal = DatasetteModal.create();
+  var dialog = document.createElement("dialog");
-  var dialog = modal.dialog;
   dialog.id = ROW_DELETE_DIALOG_ID;
   dialog.className = "row-delete-dialog";
   dialog.setAttribute("aria-labelledby", "row-delete-title");
@@ -4374,14 +4454,13 @@ function ensureRowDeleteDialog(manager) {
     <p class="row-delete-message" id="row-delete-message">Delete row <span class="row-delete-id"></span>?</p>
     <p class="row-delete-error" role="alert" hidden></p>
     <div class="modal-footer">
-      <button type="button" class="modal-btn modal-btn-ghost row-delete-cancel">Cancel</button>
+      <button type="button" class="btn btn-ghost row-delete-cancel">Cancel</button>
-      <button type="button" class="modal-btn modal-btn-primary row-delete-confirm">Delete row</button>
+      <button type="button" class="btn btn-primary row-delete-confirm">Delete row</button>
     </div>
   `;
-  document.body.appendChild(modal);
+  document.body.appendChild(dialog);
 
   rowDeleteDialogState = {
-    modal: modal,
     dialog: dialog,
     title: dialog.querySelector(".modal-title"),
     message: dialog.querySelector(".row-delete-message"),
@@ -4394,10 +4473,21 @@ function ensureRowDeleteDialog(manager) {
     currentPkPath: null,
     manager: manager,
     isBusy: false,
+    shouldRestoreFocus: true,
   };
 
   rowDeleteDialogState.cancelButton.addEventListener("click", function () {
-    modal.requestClose("cancel");
+    if (!rowDeleteDialogState.isBusy) {
+      rowDeleteDialogState.shouldRestoreFocus = true;
+      dialog.close();
+    }
+  });
+
+  dialog.addEventListener("click", function (ev) {
+    if (ev.target === dialog && !rowDeleteDialogState.isBusy) {
+      rowDeleteDialogState.shouldRestoreFocus = true;
+      dialog.close();
+    }
   });
 
   dialog.addEventListener("keydown", function (ev) {
@@ -4409,6 +4499,25 @@ function ensureRowDeleteDialog(manager) {
       if (!rowDeleteDialogState.isBusy) {
         rowDeleteDialogState.confirmButton.click();
       }
+      return;
+    }
+    if (ev.key !== "Escape") {
+      return;
+    }
+    if (rowDeleteDialogState.isBusy) {
+      ev.preventDefault();
+      return;
+    }
+    ev.preventDefault();
+    rowDeleteDialogState.shouldRestoreFocus = true;
+    dialog.close();
+  });
+
+  dialog.addEventListener("cancel", function (ev) {
+    if (rowDeleteDialogState.isBusy) {
+      ev.preventDefault();
+    } else {
+      rowDeleteDialogState.shouldRestoreFocus = true;
     }
   });
 
@@ -4416,6 +4525,13 @@ function ensureRowDeleteDialog(manager) {
     var state = rowDeleteDialogState;
     clearRowDeleteDialogError(state);
     setRowDeleteDialogBusy(state, false);
+    if (
+      state.shouldRestoreFocus &&
+      state.currentButton &&
+      document.contains(state.currentButton)
+    ) {
+      state.currentButton.focus();
+    }
   });
 
   rowDeleteDialogState.confirmButton.addEventListener(
@@ -4442,7 +4558,8 @@ function ensureRowDeleteDialog(manager) {
           throw rowMutationRequestError(response, data);
         }
         if (data && data.redirect) {
-          state.modal.close({ restoreFocus: false });
+          state.shouldRestoreFocus = false;
+          state.dialog.close();
           location.href = data.redirect;
           return;
         }
@@ -4454,7 +4571,8 @@ function ensureRowDeleteDialog(manager) {
         var statusMessage = state.currentPkPath
           ? "Deleted row " + state.currentPkPath + "."
           : "Deleted row.";
-        state.modal.close({ restoreFocus: false });
+        state.shouldRestoreFocus = false;
+        state.dialog.close();
         state.currentRow.remove();
         showRowMutationStatus(state.manager, statusMessage, false);
         if (focusTarget && document.contains(focusTarget)) {
@@ -4483,9 +4601,11 @@ function openRowDeleteDialog(button, manager) {
   }
 
   state.manager = manager;
+  state.currentButton = button;
   state.currentRow = row;
   state.currentDeleteUrl = rowDeleteUrl(row);
   state.currentPkPath = rowDisplayLabel(row);
+  state.shouldRestoreFocus = true;
 
   clearRowDeleteDialogError(state);
   setRowDeleteDialogBusy(state, false);
@@ -4497,7 +4617,9 @@ function openRowDeleteDialog(button, manager) {
   );
   state.rowId.textContent = state.currentPkPath || "this row";
 
-  state.modal.show({ returnFocusTo: button });
+  if (!state.dialog.open) {
+    state.dialog.showModal();
+  }
   state.confirmButton.focus();
 }
 
@@ -5572,7 +5694,6 @@ function setRowEditDialogLoading(state, isLoading) {
 
 function setRowEditDialogSaving(state, isSaving) {
   state.isSaving = isSaving;
-  state.modal.busy = isSaving;
   updateRowEditDialogButtons(state);
 }
 
@@ -5790,6 +5911,18 @@ function confirmDiscardRowEditChanges(state) {
   return window.confirm(message);
 }
 
+function closeRowEditDialogIfConfirmed(state) {
+  if (!state || state.isSaving) {
+    return false;
+  }
+  if (!confirmDiscardRowEditChanges(state)) {
+    return false;
+  }
+  state.shouldRestoreFocus = true;
+  state.dialog.close();
+  return true;
+}
+
 function setRowInsertDialogTitle(state) {
   var insertData = tableInsertData() || {};
   var title = rowEditIsMultipleInsert(state)
@@ -6615,6 +6748,38 @@ async function insertBulkPreviewRows(state) {
   }
 }
 
+function scheduleCloseRowEditDialogIfConfirmed(state) {
+  // Fix for an issue in Safari where hitting Esc would show
+  // the confirm() prompt asking if state should be discarded
+  // but the Esc key press would then cancel that dialog too.
+  // Wait for keyup, then move the confirm() to a fresh timer tick.
+  if (!state || state.isSaving || state.isClosePending) {
+    return false;
+  }
+  if (!rowEditDialogHasChanges(state)) {
+    state.shouldRestoreFocus = true;
+    state.dialog.close();
+    return true;
+  }
+  state.isClosePending = true;
+  var closeAfterKeyup = function () {
+    if (!state.isClosePending) {
+      return;
+    }
+    state.isClosePending = false;
+    closeRowEditDialogIfConfirmed(state);
+  };
+  var onKeyup = function (ev) {
+    if (ev.key !== "Escape") {
+      return;
+    }
+    document.removeEventListener("keyup", onKeyup, true);
+    setTimeout(closeAfterKeyup, 0);
+  };
+  document.addEventListener("keyup", onKeyup, true);
+  return true;
+}
+
 function findDataRowElement(root, rowId) {
   var elements = root.querySelectorAll("[data-row]");
   for (var i = 0; i < elements.length; i += 1) {
@@ -6704,8 +6869,9 @@ async function saveRowEditDialog(state) {
     }
     var formValues = collectRowFormValues(state);
     if (state.mode === "edit" && !Object.keys(formValues).length) {
+      state.shouldRestoreFocus = true;
       hideRowMutationStatus();
-      state.modal.close();
+      state.dialog.close();
       return;
     }
     var payload =
@@ -6738,8 +6904,9 @@ async function saveRowEditDialog(state) {
         insertedRowData,
         insertData.primaryKeys || [],
       );
+      state.shouldRestoreFocus = false;
       if (!insertedRowId) {
-        state.modal.close({ restoreFocus: false });
+        state.dialog.close();
         var missingIdStatus = showRowMutationStatus(
           state.manager,
           "Inserted row. Refresh the page to see it.",
@@ -6755,7 +6922,7 @@ async function saveRowEditDialog(state) {
       try {
         insertedRow = await fetchUpdatedRowElement(state);
       } catch (_error) {
-        state.modal.close({ restoreFocus: false });
+        state.dialog.close();
         var refreshFailedStatus = showRowMutationStatus(
           state.manager,
           "Inserted row, but could not refresh the table row. Refresh the page to see it.",
@@ -6770,7 +6937,7 @@ async function saveRowEditDialog(state) {
           rowTitleLabel(insertedRow),
         );
         var addedRow = addInsertedRowToPage(insertedRow);
-        state.modal.close({ restoreFocus: false });
+        state.dialog.close();
         showRowMutationStatus(state.manager, insertedStatusMessage, false);
         if (addedRow) {
           var insertedFocusTarget =
@@ -6779,7 +6946,7 @@ async function saveRowEditDialog(state) {
           insertedFocusTarget.focus();
         }
       } else {
-        state.modal.close({ restoreFocus: false });
+        state.dialog.close();
         var filteredStatus = showRowMutationStatus(
           state.manager,
           "Inserted row. It does not match the current filters.",
@@ -6791,7 +6958,8 @@ async function saveRowEditDialog(state) {
     }
 
     if (isRowPage()) {
-      state.modal.close({ restoreFocus: false });
+      state.shouldRestoreFocus = false;
+      state.dialog.close();
       location.reload();
       return;
     }
@@ -6827,7 +6995,8 @@ async function saveRowEditDialog(state) {
       );
     }
 
-    state.modal.close({ restoreFocus: false });
+    state.shouldRestoreFocus = false;
+    state.dialog.close();
     if (focusTarget && document.contains(focusTarget)) {
       focusTarget.focus();
     }
@@ -6971,8 +7140,7 @@ function ensureRowEditDialog(manager) {
     return null;
   }
 
-  var modal = DatasetteModal.create();
+  var dialog = document.createElement("dialog");
-  var dialog = modal.dialog;
   dialog.id = ROW_EDIT_DIALOG_ID;
   dialog.className = "row-edit-dialog";
   dialog.setAttribute("aria-labelledby", "row-edit-title");
@@ -6984,8 +7152,8 @@ function ensureRowEditDialog(manager) {
       <p class="row-edit-summary" id="row-edit-summary" hidden></p>
       <p class="row-edit-loading" role="status" aria-live="polite">Loading row...</p>
       <p class="row-edit-error" role="alert" tabindex="-1" hidden></p>
-      <div class="modal-body row-edit-fields"></div>
+      <div class="row-edit-fields"></div>
-      <div class="modal-body row-edit-bulk" hidden>
+      <div class="row-edit-bulk" hidden>
         <div class="row-edit-bulk-editor">
           <p class="row-edit-bulk-note"><label for="row-edit-bulk-textarea">Paste TSV, CSV, or JSON</label>. You can also <button type="button" class="button-as-link row-edit-bulk-open-file">open a file</button> or drop it onto this textarea</p>
           <input class="row-edit-bulk-file-input" type="file" accept=".csv,.tsv,.json,.txt,text/csv,text/tab-separated-values,application/json,text/plain" hidden>
@@ -7002,7 +7170,7 @@ function ensureRowEditDialog(manager) {
             </div>
           </div>
           <div class="row-edit-bulk-actions">
-            <button type="button" class="modal-btn modal-btn-ghost row-edit-copy-template"><span class="row-edit-copy-template-label-wide">Copy spreadsheet template</span><span class="row-edit-copy-template-label-narrow">Copy template</span></button>
+            <button type="button" class="btn btn-ghost row-edit-copy-template"><span class="row-edit-copy-template-label-wide">Copy spreadsheet template</span><span class="row-edit-copy-template-label-narrow">Copy template</span></button>
             <span class="row-edit-bulk-template-note"><span class="row-edit-bulk-template-note-wide">You can paste the template into Google Sheets or Excel.</span><span class="row-edit-bulk-template-note-narrow">Paste into Google Sheets or Excel</span></span>
           </div>
         </div>
@@ -7015,15 +7183,14 @@ function ensureRowEditDialog(manager) {
       <div class="modal-footer">
         <a href="#" class="row-edit-mode-link row-edit-bulk-insert" hidden>Insert multiple rows</a>
         <a href="#" class="row-edit-mode-link row-edit-single-insert" hidden>Insert single row</a>
-        <button type="button" class="modal-btn modal-btn-ghost row-edit-cancel">Cancel</button>
+        <button type="button" class="btn btn-ghost row-edit-cancel">Cancel</button>
-        <button type="submit" class="modal-btn modal-btn-primary row-edit-save" disabled>Save</button>
+        <button type="submit" class="btn btn-primary row-edit-save" disabled>Save</button>
       </div>
     </form>
   `;
-  document.body.appendChild(modal);
+  document.body.appendChild(dialog);
 
   rowEditDialogState = {
-    modal: modal,
     dialog: dialog,
     form: dialog.querySelector(".row-edit-form"),
     title: dialog.querySelector(".modal-title"),
@@ -7054,6 +7221,7 @@ function ensureRowEditDialog(manager) {
     singleInsertLink: dialog.querySelector(".row-edit-single-insert"),
     cancelButton: dialog.querySelector(".row-edit-cancel"),
     saveButton: dialog.querySelector(".row-edit-save"),
+    currentButton: null,
     currentRow: null,
     currentRowId: null,
     currentPkPath: null,
@@ -7081,7 +7249,9 @@ function ensureRowEditDialog(manager) {
     manager: manager,
     isLoading: false,
     isSaving: false,
+    isClosePending: false,
     hasLoaded: false,
+    shouldRestoreFocus: true,
   };
 
   rowEditDialogState.form.addEventListener("submit", function (ev) {
@@ -7101,7 +7271,10 @@ function ensureRowEditDialog(manager) {
       rowEditDialogState.bulkInsertTextarea.focus();
       return;
     }
-    modal.requestClose("cancel");
+    if (!rowEditDialogState.isSaving) {
+      rowEditDialogState.shouldRestoreFocus = true;
+      dialog.close();
+    }
   });
 
   rowEditDialogState.bulkInsertLink.addEventListener("click", function (ev) {
@@ -7220,17 +7393,31 @@ function ensureRowEditDialog(manager) {
     },
   );
 
-  modal.beforeClose = function (source) {
+  dialog.addEventListener("click", function (ev) {
-    return (
+    if (ev.target === dialog) {
-      source === "cancel" || confirmDiscardRowEditChanges(rowEditDialogState)
+      closeRowEditDialogIfConfirmed(rowEditDialogState);
-    );
+    }
-  };
+  });
+
+  dialog.addEventListener("keydown", function (ev) {
+    if (ev.key !== "Escape") {
+      return;
+    }
+    ev.preventDefault();
+    scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState);
+  });
+
+  dialog.addEventListener("cancel", function (ev) {
+    ev.preventDefault();
+    scheduleCloseRowEditDialogIfConfirmed(rowEditDialogState);
+  });
 
   dialog.addEventListener("close", function () {
     var state = rowEditDialogState;
     var shouldReloadOnClose = state.shouldReloadOnClose;
     var redirectOnCloseUrl = state.redirectOnCloseUrl;
     state.loadId += 1;
+    state.isClosePending = false;
     state.bulkInsertLiveValidationError = null;
     state.shouldReloadOnClose = false;
     state.redirectOnCloseUrl = null;
@@ -7243,6 +7430,13 @@ function ensureRowEditDialog(manager) {
     destroyRowEditFields(state);
     setRowEditDialogLoading(state, false);
     setRowEditDialogSaving(state, false);
+    if (
+      state.shouldRestoreFocus &&
+      state.currentButton &&
+      document.contains(state.currentButton)
+    ) {
+      state.currentButton.focus();
+    }
     if (shouldReloadOnClose) {
       if (redirectOnCloseUrl) {
         location.href = redirectOnCloseUrl;
@@ -7267,6 +7461,7 @@ async function openRowEditDialog(button, manager) {
 
   state.manager = manager;
   state.mode = "edit";
+  state.currentButton = button;
   state.currentRow = row;
   state.currentRowId = row.getAttribute("data-row") || "";
   state.currentPkPath = rowDisplayLabel(row);
@@ -7283,7 +7478,7 @@ async function openRowEditDialog(button, manager) {
   } else {
     state.form.removeAttribute("action");
   }
-
+  state.shouldRestoreFocus = true;
   state.hasLoaded = false;
   state.loadId += 1;
   var loadId = state.loadId;
@@ -7302,7 +7497,9 @@ async function openRowEditDialog(button, manager) {
   state.summary.textContent = "";
   syncRowEditInsertModeUi(state);
 
-  state.modal.show({ returnFocusTo: button });
+  if (!state.dialog.open) {
+    state.dialog.showModal();
+  }
   state.cancelButton.focus();
 
   try {
@@ -7342,6 +7539,7 @@ function openRowInsertDialog(button, manager) {
 
   state.manager = manager;
   state.mode = "insert";
+  state.currentButton = button;
   state.currentRow = null;
   state.currentRowId = null;
   state.currentPkPath = null;
@@ -7356,7 +7554,7 @@ function openRowInsertDialog(button, manager) {
   state.shouldReloadOnClose = false;
   state.redirectOnCloseUrl = null;
   resetBulkInsertPreview(state);
-
+  state.shouldRestoreFocus = true;
   state.hasLoaded = false;
   state.loadId += 1;
 
@@ -7378,7 +7576,9 @@ function openRowInsertDialog(button, manager) {
   state.summary.textContent = "";
   syncRowEditInsertModeUi(state);
 
-  state.modal.show({ returnFocusTo: button });
+  if (!state.dialog.open) {
+    state.dialog.showModal();
+  }
   renderRowInsertFields(state, insertData);
 }
 

datasette/static/json-format-highlight-1.0.1.js

@@ -0,0 +1,56 @@
+/*
+https://github.com/luyilin/json-format-highlight
+From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js
+MIT Licensed
+*/
+(function (global, factory) {
+  typeof exports === "object" && typeof module !== "undefined"
+    ? (module.exports = factory())
+    : typeof define === "function" && define.amd
+      ? define(factory)
+      : (global.jsonFormatHighlight = factory());
+})(this, function () {
+  "use strict";
+
+  var defaultColors = {
+    keyColor: "dimgray",
+    numberColor: "lightskyblue",
+    stringColor: "lightcoral",
+    trueColor: "lightseagreen",
+    falseColor: "#f66578",
+    nullColor: "cornflowerblue",
+  };
+
+  function index(json, colorOptions) {
+    if (colorOptions === void 0) colorOptions = {};
+
+    if (!json) {
+      return;
+    }
+    if (typeof json !== "string") {
+      json = JSON.stringify(json, null, 2);
+    }
+    var colors = Object.assign({}, defaultColors, colorOptions);
+    json = json.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
+    return json.replace(
+      /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g,
+      function (match) {
+        var color = colors.numberColor;
+        if (/^"/.test(match)) {
+          color = /:$/.test(match) ? colors.keyColor : colors.stringColor;
+        } else {
+          color = /true/.test(match)
+            ? colors.trueColor
+            : /false/.test(match)
+              ? colors.falseColor
+              : /null/.test(match)
+                ? colors.nullColor
+                : color;
+        }
+        return '<span style="color: ' + color + '">' + match + "</span>";
+      },
+    );
+  }
+
+  return index;
+});

datasette/static/mobile-column-actions.js

@@ -66,8 +66,7 @@ function initMobileColumnActions(manager) {
     return;
   }
 
-  var modal = DatasetteModal.create();
+  var dialog = document.createElement("dialog");
-  var dialog = modal.dialog;
   dialog.className = "mobile-column-actions-dialog";
   dialog.id = MOBILE_COLUMN_DIALOG_ID;
   dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID);
@@ -76,13 +75,13 @@ function initMobileColumnActions(manager) {
       <span class="modal-title" id="${MOBILE_COLUMN_DIALOG_TITLE_ID}">Column actions</span>
       <span class="modal-meta"></span>
     </div>
-    <div class="modal-body list-wrap mobile-column-list"></div>
+    <div class="list-wrap mobile-column-list"></div>
     <div class="modal-footer">
       <span class="footer-info">Tap a column to reveal actions.</span>
-      <button type="button" class="modal-btn modal-btn-ghost mobile-column-actions-done">Done</button>
+      <button type="button" class="btn btn-ghost mobile-column-actions-done">Done</button>
     </div>
   `;
-  document.body.appendChild(modal);
+  document.body.appendChild(dialog);
 
   triggerButton.setAttribute("aria-haspopup", "dialog");
   triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID);
@@ -92,6 +91,7 @@ function initMobileColumnActions(manager) {
   var listWrap = dialog.querySelector(".mobile-column-list");
   var doneButton = dialog.querySelector(".mobile-column-actions-done");
   var expandedSectionId = null;
+  var shouldRestoreFocus = true;
 
   function updateExpandedSection() {
     Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => {
@@ -128,7 +128,16 @@ function initMobileColumnActions(manager) {
   }
 
   function closeDialog(options) {
-    modal.close(options);
+    options = options || {};
+    shouldRestoreFocus = options.restoreFocus !== false;
+    if (dialog.open) {
+      dialog.close();
+    } else {
+      triggerButton.setAttribute("aria-expanded", "false");
+      if (shouldRestoreFocus) {
+        triggerButton.focus();
+      }
+    }
   }
 
   function renderDialog() {
@@ -157,8 +166,7 @@ function initMobileColumnActions(manager) {
       topActions.className = "mobile-column-top-actions";
 
       var showAllColumns = document.createElement("a");
-      showAllColumns.className =
+      showAllColumns.className = "btn btn-ghost mobile-column-top-action";
-        "modal-btn modal-btn-ghost mobile-column-top-action";
       showAllColumns.href = manager.columnActions.showAllColumnsUrl();
       showAllColumns.textContent = "Show all columns";
 
@@ -257,7 +265,9 @@ function initMobileColumnActions(manager) {
     if (!renderDialog()) {
       return;
     }
-    modal.show({ returnFocusTo: triggerButton });
+    if (!dialog.open) {
+      dialog.showModal();
+    }
     triggerButton.setAttribute("aria-expanded", "true");
     var focusTarget =
       dialog.querySelector(".mobile-column-top-action") ||
@@ -278,8 +288,22 @@ function initMobileColumnActions(manager) {
     closeDialog();
   });
 
+  dialog.addEventListener("click", function (ev) {
+    if (ev.target === dialog) {
+      closeDialog();
+    }
+  });
+
+  dialog.addEventListener("cancel", function (ev) {
+    ev.preventDefault();
+    closeDialog();
+  });
+
   dialog.addEventListener("close", function () {
     triggerButton.setAttribute("aria-expanded", "false");
+    if (shouldRestoreFocus) {
+      triggerButton.focus();
+    }
   });
 
   window.addEventListener("resize", function () {

datasette/static/modal.js

@@ -1,167 +0,0 @@
-// Shared lifecycle for native modal dialogs.
-(() => {
-  class DatasetteModal extends HTMLElement {
-    constructor() {
-      super();
-      this.beforeClose = null;
-      this._busy = false;
-      this._restoreFocus = true;
-      this._returnFocusTo = null;
-      this._escapeCleanup = null;
-      this._escapeTimer = null;
-    }
-
-    static create() {
-      const modal = document.createElement("datasette-modal");
-      modal.appendChild(document.createElement("dialog"));
-      return modal;
-    }
-
-    get dialog() {
-      return this.querySelector(":scope > dialog");
-    }
-
-    get busy() {
-      return this._busy;
-    }
-
-    set busy(value) {
-      this._busy = !!value;
-      if (this.dialog) {
-        this.dialog.setAttribute("aria-busy", String(this._busy));
-      }
-    }
-
-    connectedCallback() {
-      const dialog = this.dialog;
-      if (!dialog) return;
-      dialog.classList.add("datasette-modal");
-      this._listeners?.abort();
-      this._listeners = new AbortController();
-      const options = { signal: this._listeners.signal };
-      let backdropPointerDown = false;
-      const outside = (event) => {
-        const rect = dialog.getBoundingClientRect();
-        return (
-          event.target === dialog &&
-          (event.clientX < rect.left ||
-            event.clientX > rect.right ||
-            event.clientY < rect.top ||
-            event.clientY > rect.bottom)
-        );
-      };
-      dialog.addEventListener(
-        "pointerdown",
-        (event) => {
-          backdropPointerDown = outside(event);
-        },
-        options,
-      );
-      dialog.addEventListener(
-        "click",
-        (event) => {
-          if (backdropPointerDown && outside(event))
-            this.requestClose("backdrop");
-          backdropPointerDown = false;
-        },
-        options,
-      );
-      dialog.addEventListener(
-        "keydown",
-        (event) => {
-          if (event.key !== "Escape" || event.defaultPrevented) return;
-          // A nested native dialog or plugin picker gets first refusal.
-          if (event.target.closest("dialog") !== dialog) return;
-          event.preventDefault();
-          if (this.busy || this._escapeCleanup || this._escapeTimer !== null)
-            return;
-          // Safari can otherwise use this Escape press to cancel confirm() too.
-          // Only keyboard dismissals wait for keyup; native cancel events needn't.
-          const onKeyup = (up) => {
-            if (up.key !== "Escape") return;
-            this._escapeCleanup();
-            this._escapeCleanup = null;
-            this._escapeTimer = setTimeout(() => {
-              this._escapeTimer = null;
-              this.requestClose("escape");
-            }, 0);
-          };
-          this.ownerDocument.addEventListener("keyup", onKeyup, true);
-          this._escapeCleanup = () =>
-            this.ownerDocument.removeEventListener("keyup", onKeyup, true);
-        },
-        options,
-      );
-      dialog.addEventListener(
-        "cancel",
-        (event) => {
-          if (event.target !== dialog) return;
-          event.preventDefault();
-          if (!this._escapeCleanup && this._escapeTimer === null)
-            this.requestClose("escape");
-        },
-        options,
-      );
-      dialog.addEventListener(
-        "close",
-        (event) => {
-          if (event.target !== dialog || dialog.open) return;
-          this._clearPendingClose();
-          this.busy = false;
-          if (this._restoreFocus && this._returnFocusTo?.isConnected) {
-            // Menu actions may have become hidden while the dialog was open.
-            const details = this._returnFocusTo.closest("details:not([open])");
-            const target =
-              details?.querySelector("summary") || this._returnFocusTo;
-            target.focus({ preventScroll: true });
-          }
-          this._returnFocusTo = null;
-        },
-        options,
-      );
-    }
-
-    disconnectedCallback() {
-      this._listeners?.abort();
-      this._clearPendingClose();
-      this._returnFocusTo = null;
-      if (this.dialog?.open) this.dialog.close();
-      this.busy = false;
-    }
-
-    _clearPendingClose() {
-      this._escapeCleanup?.();
-      this._escapeCleanup = null;
-      clearTimeout(this._escapeTimer);
-      this._escapeTimer = null;
-    }
-
-    show({ returnFocusTo, initialFocus } = {}) {
-      const dialog = this.dialog;
-      if (!dialog.open) {
-        this._clearPendingClose();
-        this._returnFocusTo = returnFocusTo || this.ownerDocument.activeElement;
-        this._restoreFocus = true;
-        dialog.showModal();
-      }
-      if (typeof initialFocus === "function") initialFocus();
-      else initialFocus?.focus();
-    }
-
-    requestClose(source = "cancel") {
-      if (!this.dialog.open || this.busy) return false;
-      if (this.beforeClose && this.beforeClose(source) === false) return false;
-      this.close();
-      return true;
-    }
-
-    close({ restoreFocus = true } = {}) {
-      this._clearPendingClose();
-      this._restoreFocus = restoreFocus;
-      this.dialog.close();
-    }
-  }
-
-  customElements.define("datasette-modal", DatasetteModal);
-  window.DatasetteModal = DatasetteModal;
-})();

datasette/static/navigation-search.js

@@ -10,22 +10,277 @@ class NavigationSearch extends HTMLElement {
     this.recentHeadingId = `navigation-search-recent-${this.instanceId}`;
     this.statusId = `navigation-search-status-${this.instanceId}`;
     this.titleId = `navigation-search-title-${this.instanceId}`;
+    this.attachShadow({ mode: "open" });
     this.selectedIndex = -1;
     this.matches = [];
     this.renderedMatches = [];
     this.debounceTimer = null;
-  }
+    this.restoreFocusTarget = null;
+    this.shouldRestoreFocus = true;
 
-  connectedCallback() {
-    if (this._initialized) return;
-    this._initialized = true;
     this.render();
     this.setupEventListeners();
   }
 
   render() {
-    this.innerHTML = `
+    this.shadowRoot.innerHTML = `
-      <datasette-modal><dialog aria-modal="true" aria-labelledby="${this.titleId}">
+            <style>
+                :host {
+                    display: contents;
+                }
+
+                dialog {
+                    border: none;
+                    border-radius: var(--modal-border-radius, 0.75rem);
+                    padding: 0;
+                    max-width: 90vw;
+                    width: 600px;
+                    max-height: 80vh;
+                    box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
+                    animation: slideIn var(--modal-animation-duration, 0.2s) ease-out;
+                }
+
+                dialog::backdrop {
+                    background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
+                    backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+                    -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+                    animation: fadeIn var(--modal-animation-duration, 0.2s) ease-out;
+                }
+
+                @keyframes slideIn {
+                    from {
+                        opacity: 0;
+                        transform: translateY(-20px) scale(0.95);
+                    }
+                    to {
+                        opacity: 1;
+                        transform: translateY(0) scale(1);
+                    }
+                }
+
+                @keyframes fadeIn {
+                    from { opacity: 0; }
+                    to { opacity: 1; }
+                }
+
+                .search-container {
+                    display: flex;
+                    flex-direction: column;
+                }
+
+                .search-input-wrapper {
+                    padding: 1.25rem;
+                    border-bottom: 1px solid #e5e7eb;
+                    display: flex;
+                    gap: 0.5rem;
+                    align-items: center;
+                }
+
+                .search-input {
+                    width: 100%;
+                    flex: 1;
+                    min-width: 0;
+                    padding: 0.75rem 1rem;
+                    font-size: 1rem;
+                    border: 2px solid #e5e7eb;
+                    border-radius: 0.5rem;
+                    outline: none;
+                    transition: border-color 0.2s;
+                    box-sizing: border-box;
+                }
+
+                .search-input:focus {
+                    border-color: #2563eb;
+                }
+
+                .close-search {
+                    background: transparent;
+                    border: 1px solid transparent;
+                    border-radius: 0.375rem;
+                    color: #4b5563;
+                    cursor: pointer;
+                    flex: 0 0 auto;
+                    font: inherit;
+                    font-size: 1.5rem;
+                    height: 2.75rem;
+                    line-height: 1;
+                    width: 2.75rem;
+                }
+
+                .close-search:hover,
+                .close-search:focus {
+                    background-color: #f3f4f6;
+                    border-color: #d1d5db;
+                }
+
+                .results-container {
+                    overflow-y: auto;
+                    height: calc(80vh - 180px);
+                    padding: 0.5rem;
+                }
+
+                .results-list:empty {
+                    display: none;
+                }
+
+                .result-item {
+                    padding: 0.875rem 1rem;
+                    cursor: pointer;
+                    border-radius: 0.5rem;
+                    transition: background-color 0.15s;
+                    display: flex;
+                    align-items: center;
+                    gap: 0.75rem;
+                }
+
+                .result-item:hover {
+                    background-color: #f3f4f6;
+                }
+
+                .result-item.selected {
+                    background-color: #dbeafe;
+                }
+
+                .result-item > div {
+                    flex: 1;
+                    min-width: 0;
+                }
+
+                .jump-start-content {
+                    border-bottom: 1px solid #e5e7eb;
+                    margin-bottom: 0.5rem;
+                    padding: 0.5rem 0.5rem 1rem;
+                }
+
+                .jump-start-content:empty {
+                    display: none;
+                }
+
+                .result-name {
+                    font-weight: 500;
+                    color: #111827;
+                }
+
+                .result-label {
+                    font-size: 0.875rem;
+                    color: #4b5563;
+                }
+
+                .result-type {
+                    color: #4b5563;
+                    font-size: 0.75rem;
+                    font-weight: 600;
+                    text-transform: uppercase;
+                }
+
+                .result-url {
+                    font-size: 0.875rem;
+                    color: #6b7280;
+                }
+
+                .result-description {
+                    color: #374151;
+                    display: -webkit-box;
+                    font-size: 0.8125rem;
+                    line-height: 1.35;
+                    margin-top: 0.35rem;
+                    overflow: hidden;
+                    -webkit-box-orient: vertical;
+                    -webkit-line-clamp: 2;
+                }
+
+                .results-heading {
+                    color: #4b5563;
+                    font-size: 0.75rem;
+                    font-weight: 600;
+                    letter-spacing: 0;
+                    padding: 0.5rem 1rem 0.25rem;
+                    text-transform: uppercase;
+                }
+
+                .recent-actions {
+                    padding: 0.25rem 1rem 0.75rem;
+                }
+
+                .clear-recent {
+                    background: transparent;
+                    border: 0;
+                    color: #2563eb;
+                    cursor: pointer;
+                    font: inherit;
+                    font-size: 0.875rem;
+                    padding: 0;
+                }
+
+                .clear-recent:hover {
+                    text-decoration: underline;
+                }
+
+                .no-results {
+                    padding: 2rem;
+                    text-align: center;
+                    color: #6b7280;
+                }
+
+                .hint-text {
+                    padding: 0.75rem 1.25rem;
+                    font-size: 0.875rem;
+                    color: #6b7280;
+                    border-top: 1px solid #e5e7eb;
+                    display: flex;
+                    gap: 1rem;
+                    flex-wrap: wrap;
+                }
+
+                .hint-text kbd {
+                    background: #f3f4f6;
+                    padding: 0.125rem 0.375rem;
+                    border-radius: 0.25rem;
+                    font-size: 0.75rem;
+                    border: 1px solid #d1d5db;
+                    font-family: monospace;
+                }
+
+                .visually-hidden {
+                    border: 0;
+                    clip: rect(0 0 0 0);
+                    height: 1px;
+                    margin: -1px;
+                    overflow: hidden;
+                    padding: 0;
+                    position: absolute;
+                    white-space: nowrap;
+                    width: 1px;
+                }
+
+                /* Mobile optimizations */
+                @media (max-width: 640px) {
+                    dialog {
+                        width: 95vw;
+                        max-height: 85vh;
+                        border-radius: 0.5rem;
+                    }
+
+                    .search-input-wrapper {
+                        padding: 1rem;
+                    }
+
+                    .search-input {
+                        font-size: 16px; /* Prevents zoom on iOS */
+                    }
+
+                    .result-item {
+                        padding: 1rem 0.75rem;
+                    }
+
+                    .hint-text {
+                        font-size: 0.8rem;
+                        padding: 0.5rem 1rem;
+                    }
+                }
+            </style>
+
+            <dialog aria-modal="true" aria-labelledby="${this.titleId}">
                 <div class="search-container">
                     <h2 id="${this.titleId}" class="visually-hidden">Jump to</h2>
                     <p id="${this.instructionsId}" class="visually-hidden">Type to search. Use up and down arrow keys to move through results, Enter to select a result, and Escape to close this menu.</p>
@@ -47,22 +302,23 @@ class NavigationSearch extends HTMLElement {
                         >
                         <button type="button" class="close-search" aria-label="Close jump menu">&times;</button>
                     </div>
-                    <div class="modal-body results-container"></div>
+                    <div class="results-container"></div>
                     <div class="hint-text">
                         <span><kbd>↑</kbd> <kbd>↓</kbd> Navigate</span>
                         <span><kbd>Enter</kbd> Select</span>
                         <span><kbd>Esc</kbd> Close</span>
                     </div>
                 </div>
-            </dialog></datasette-modal>
+            </dialog>
         `;
   }
 
   setupEventListeners() {
-    const dialog = this.querySelector("dialog");
+    const dialog = this.shadowRoot.querySelector("dialog");
-    const input = this.querySelector(".search-input");
+    const input = this.shadowRoot.querySelector(".search-input");
-    const closeButton = this.querySelector(".close-search");
+    const closeButton = this.shadowRoot.querySelector(".close-search");
-    const resultsContainer = this.querySelector(".results-container");
+    const resultsContainer =
+      this.shadowRoot.querySelector(".results-container");
 
     // Global keyboard listener for "/"
     document.addEventListener("keydown", (e) => {
@@ -99,6 +355,8 @@ class NavigationSearch extends HTMLElement {
       } else if (e.key === "Enter") {
         e.preventDefault();
         this.selectCurrentItem();
+      } else if (e.key === "Escape") {
+        this.closeMenu();
       }
     });
 
@@ -122,6 +380,18 @@ class NavigationSearch extends HTMLElement {
       }
     });
 
+    // Close on backdrop click
+    dialog.addEventListener("click", (e) => {
+      if (e.target === dialog) {
+        this.closeMenu();
+      }
+    });
+
+    dialog.addEventListener("cancel", (e) => {
+      e.preventDefault();
+      this.closeMenu();
+    });
+
     dialog.addEventListener("close", () => {
       this.onMenuClosed();
     });
@@ -162,6 +432,19 @@ class NavigationSearch extends HTMLElement {
     }
   }
 
+  focusRestoreTarget(trigger) {
+    if (trigger && typeof trigger.focus === "function") {
+      return trigger;
+    }
+    if (
+      document.activeElement &&
+      typeof document.activeElement.focus === "function"
+    ) {
+      return document.activeElement;
+    }
+    return null;
+  }
+
   setNavigationTriggersExpanded(expanded) {
     if (typeof document.querySelectorAll !== "function") {
       return;
@@ -182,8 +465,8 @@ class NavigationSearch extends HTMLElement {
   }
 
   updateComboboxState() {
-    const dialog = this.querySelector("dialog");
+    const dialog = this.shadowRoot.querySelector("dialog");
-    const input = this.querySelector(".search-input");
+    const input = this.shadowRoot.querySelector(".search-input");
     const matches = this.renderedMatches || [];
     this.setElementAttribute(
       input,
@@ -208,7 +491,7 @@ class NavigationSearch extends HTMLElement {
   }
 
   setStatus(message) {
-    const status = this.querySelector(`#${this.statusId}`);
+    const status = this.shadowRoot.querySelector(`#${this.statusId}`);
     if (status) {
       status.textContent = message || "";
     }
@@ -418,7 +701,7 @@ class NavigationSearch extends HTMLElement {
       section.render(node, {
         navigationSearch: this,
         container,
-        input: this.querySelector(".search-input"),
+        input: this.shadowRoot.querySelector(".search-input"),
       });
     });
   }
@@ -457,8 +740,8 @@ class NavigationSearch extends HTMLElement {
   }
 
   renderResults() {
-    const container = this.querySelector(".results-container");
+    const container = this.shadowRoot.querySelector(".results-container");
-    const input = this.querySelector(".search-input");
+    const input = this.shadowRoot.querySelector(".search-input");
     const showStartContent = !input.value.trim();
     const jumpSections = showStartContent ? this.jumpSections() : [];
     const startBlock = showStartContent
@@ -570,15 +853,18 @@ class NavigationSearch extends HTMLElement {
     }
   }
 
-  openMenu(returnFocusTo) {
+  openMenu(trigger) {
-    const input = this.querySelector(".search-input");
+    const dialog = this.shadowRoot.querySelector("dialog");
+    const input = this.shadowRoot.querySelector(".search-input");
 
-    this.querySelector("datasette-modal").show({
+    this.restoreFocusTarget = this.focusRestoreTarget(trigger);
-      returnFocusTo,
+    this.shouldRestoreFocus = true;
-      initialFocus: input,
+    if (!dialog.open) {
-    });
+      dialog.showModal();
+    }
     this.setNavigationTriggersExpanded(true);
     input.value = "";
+    input.focus();
 
     // Reset state, then populate the default jump list.
     this.matches = [];
@@ -588,15 +874,29 @@ class NavigationSearch extends HTMLElement {
   }
 
   closeMenu(options = {}) {
-    this.querySelector("datasette-modal").close(options);
+    const dialog = this.shadowRoot.querySelector("dialog");
+    this.shouldRestoreFocus = options.restoreFocus !== false;
+    if (dialog.open) {
+      dialog.close();
+    } else {
+      this.onMenuClosed();
+    }
   }
 
   onMenuClosed() {
-    const input = this.querySelector(".search-input");
+    const input = this.shadowRoot.querySelector(".search-input");
     this.setElementAttribute(input, "aria-expanded", "false");
     this.removeElementAttribute(input, "aria-activedescendant");
     this.setNavigationTriggersExpanded(false);
     this.setStatus("");
+    if (
+      this.shouldRestoreFocus &&
+      this.restoreFocusTarget &&
+      typeof this.restoreFocusTarget.focus === "function"
+    ) {
+      this.restoreFocusTarget.focus();
+    }
+    this.restoreFocusTarget = null;
   }
 
   escapeHtml(text) {

datasette/static/table.js

@@ -157,7 +157,6 @@ function createSetColumnTypeOption(value, name, description, checked) {
 
 function setSetColumnTypeDialogBusy(state, isBusy) {
   state.isBusy = isBusy;
-  state.modal.busy = isBusy;
   state.saveButton.disabled = isBusy;
   state.cancelButton.disabled = isBusy;
   Array.from(
@@ -186,8 +185,7 @@ function ensureSetColumnTypeDialog() {
     return null;
   }
 
-  var modal = DatasetteModal.create();
+  var dialog = document.createElement("dialog");
-  var dialog = modal.dialog;
   dialog.id = SET_COLUMN_TYPE_DIALOG_ID;
   dialog.className = "set-column-type-dialog";
   dialog.setAttribute("aria-labelledby", "set-column-type-title");
@@ -198,17 +196,16 @@ function ensureSetColumnTypeDialog() {
     </div>
     <p class="set-column-type-status"></p>
     <p class="set-column-type-error" hidden></p>
-    <div class="modal-body set-column-type-options"></div>
+    <div class="set-column-type-options"></div>
     <div class="modal-footer">
       <span class="footer-info"></span>
-      <button type="button" class="modal-btn modal-btn-ghost set-column-type-cancel">Cancel</button>
+      <button type="button" class="btn btn-ghost set-column-type-cancel">Cancel</button>
-      <button type="button" class="modal-btn modal-btn-primary set-column-type-save">Save</button>
+      <button type="button" class="btn btn-primary set-column-type-save">Save</button>
     </div>
   `;
-  document.body.appendChild(modal);
+  document.body.appendChild(dialog);
 
   setColumnTypeDialogState = {
-    modal: modal,
     dialog: dialog,
     meta: dialog.querySelector(".modal-meta"),
     status: dialog.querySelector(".set-column-type-status"),
@@ -223,7 +220,21 @@ function ensureSetColumnTypeDialog() {
   };
 
   setColumnTypeDialogState.cancelButton.addEventListener("click", function () {
-    modal.requestClose("cancel");
+    if (!setColumnTypeDialogState.isBusy) {
+      dialog.close();
+    }
+  });
+
+  dialog.addEventListener("click", function (ev) {
+    if (ev.target === dialog && !setColumnTypeDialogState.isBusy) {
+      dialog.close();
+    }
+  });
+
+  dialog.addEventListener("cancel", function (ev) {
+    if (setColumnTypeDialogState.isBusy) {
+      ev.preventDefault();
+    }
   });
 
   dialog.addEventListener("close", function () {
@@ -231,9 +242,7 @@ function ensureSetColumnTypeDialog() {
     setSetColumnTypeDialogBusy(setColumnTypeDialogState, false);
   });
 
-  setColumnTypeDialogState.saveButton.addEventListener(
+  setColumnTypeDialogState.saveButton.addEventListener("click", async function () {
-    "click",
-    async function () {
     var state = setColumnTypeDialogState;
     var selected = state.dialog.querySelector(
       'input[name="set-column-type-choice"]:checked',
@@ -244,7 +253,7 @@ function ensureSetColumnTypeDialog() {
       : "";
 
     if (selectedType === currentType) {
-        state.modal.close();
+      state.dialog.close();
       return;
     }
 
@@ -275,8 +284,7 @@ function ensureSetColumnTypeDialog() {
       setSetColumnTypeDialogBusy(state, false);
       showSetColumnTypeDialogError(state, error.message || "Request failed");
     }
-    },
+  });
-  );
 
   return setColumnTypeDialogState;
 }
@@ -333,7 +341,9 @@ function openSetColumnTypeDialog(th) {
     state.optionsWrap.appendChild(emptyState);
   }
 
-  state.modal.show();
+  if (!state.dialog.open) {
+    state.dialog.showModal();
+  }
   var selectedOption = state.dialog.querySelector(
     'input[name="set-column-type-choice"]:checked',
   );
@@ -357,10 +367,9 @@ function shouldShowShowAllColumns() {
 
 function hasMultipleVisibleColumns(manager) {
   return (
-    Array.from(
+    Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter(
-      document.querySelectorAll(manager.selectors.tableHeaders),
+      (th) => th.dataset.column && th.dataset.isLinkColumn !== "1",
-    ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1")
+    ).length > 1
-      .length > 1
   );
 }
 
@@ -640,12 +649,10 @@ function filterRowNumberFromName(name) {
 }
 
 function nextFilterRowNumber(manager) {
-  return (
+  return filterRowsWithControls(manager).reduce((max, row) => {
-    filterRowsWithControls(manager).reduce((max, row) => {
     var column = row.querySelector("select");
     return Math.max(max, filterRowNumberFromName(column && column.name));
-    }, 0) + 1
+  }, 0) + 1;
-  );
 }
 
 function setFilterRowNumber(row, number) {
@@ -672,11 +679,9 @@ function updateFilterRowButtons(manager) {
     if (addButton) {
       addButton.hidden = index !== rows.length - 1 || !column.value;
     }
-    var visibleButtonCount = [removeButton, addButton].filter(
+    var visibleButtonCount = [removeButton, addButton].filter(function (button) {
-      function (button) {
       return button && !button.hidden;
-      },
+    }).length;
-    ).length;
     row.classList.toggle(
       "filter-controls-row-has-buttons",
       visibleButtonCount > 0,
@@ -698,9 +703,7 @@ function cloneFilterRow(row) {
   clone.querySelector(".filter-op select").name = "_filter_op";
   clone.querySelector("input.filter-value").name = "_filter_value";
   resetFilterRow(clone);
-  clone
+  clone.querySelectorAll(".filter-row-icon").forEach((button) => button.remove());
-    .querySelectorAll(".filter-row-icon")
-    .forEach((button) => button.remove());
   return clone;
 }
 
@@ -857,45 +860,10 @@ function openColumnChooser() {
   });
 }
 
-function initCountAll() {
-  var button = document.querySelector(".count-all");
-  if (!button) {
-    return;
-  }
-  button.addEventListener("click", async function () {
-    var count = document.querySelector(".table-count");
-    var error = document.querySelector(".count-error");
-    button.disabled = true;
-    button.textContent = "Counting…";
-    error.textContent = "";
-    try {
-      var response = await fetch(button.dataset.countUrl + location.search, {
-        method: "POST",
-        headers: {
-          Accept: "application/json",
-        },
-      });
-      var data = await response.json();
-      if (!response.ok || !data.ok) {
-        throw new Error((data.errors || ["Count failed"]).join(" "));
-      }
-      count.textContent =
-        data.count.toLocaleString("en-US") +
-        (data.count === 1 ? " row" : " rows");
-      button.remove();
-    } catch (ex) {
-      error.textContent = ex.message || "Count failed";
-      button.disabled = false;
-      button.textContent = "count all";
-    }
-  });
-}
-
 // Ensures Table UI is initialized only after the Manager is ready.
 document.addEventListener("datasette_init", function (evt) {
   const { detail: manager } = evt;
 
-  initCountAll();
   initializeColumnActions(manager);
 
   // Main table

datasette/telemetry.py

@@ -1,481 +0,0 @@
-"""
-OpenTelemetry integration for Datasette.
-
-This uses `opentelemetry-api` only. Providers, exporters and sampling are
-configured by whoever runs Datasette, for example `opentelemetry-instrument`.
-"""
-
-import contextvars
-import re
-import threading
-import time
-import weakref
-from contextlib import contextmanager
-
-from opentelemetry import context as otel_context_api
-from opentelemetry import metrics as otel_metrics
-from opentelemetry import trace as otel_trace
-from opentelemetry.propagate import extract
-from opentelemetry.propagators.textmap import Getter
-from opentelemetry.trace import Link, SpanKind, Status, StatusCode, get_current_span
-
-from .telemetry_registry import (
-    DB_NAMESPACE,
-    DB_SYSTEM,
-    ERROR_TYPE,
-    HTTP_REQUEST_METHOD,
-    HTTP_RESPONSE_STATUS_CODE,
-    INTERNAL_CLIENT,
-    M_CONNECTIONS_OPEN,
-    M_OPERATION_DURATION,
-    M_QUERIES_INTERRUPTED,
-    M_QUERIES_PENDING,
-    M_THREADS_LIMIT,
-    M_THREADS_QUEUE_DEPTH,
-    M_WRITE_QUEUE_DEPTH,
-    M_WRITE_QUEUE_WAIT,
-    OPERATION,
-    SERVER_ADDRESS,
-    URL_PATH,
-    URL_SCHEME,
-    USER_AGENT_ORIGINAL,
-)
-from .version import __version__
-
-# True while code is executing within a datasette.client request. Defined
-# here rather than in app.py to avoid a circular import.
-_in_datasette_client = contextvars.ContextVar("in_datasette_client", default=False)
-
-# The semantic conventions version matching the attribute names used here.
-# 1.30.0 renamed `db.system` to `db.system.name`, so update this when
-# renaming attributes to match a newer version.
-SCHEMA_URL = "https://opentelemetry.io/schemas/1.29.0"
-
-tracer = otel_trace.get_tracer("datasette", __version__, schema_url=SCHEMA_URL)
-meter = otel_metrics.get_meter("datasette", __version__, schema_url=SCHEMA_URL)
-
-MAX_SQL_LENGTH = 2048
-
-
-def sql_attribute(sql: str) -> str:
-    "Truncate SQL text so it is safe to attach to a span as an attribute."
-    sql = sql.strip()
-    if len(sql) <= MAX_SQL_LENGTH:
-        return sql
-    return sql[:MAX_SQL_LENGTH] + "…[truncated]"
-
-
-def callback_name(fn) -> str:
-    """
-    The name recorded as `datasette.callback` for a callback-style call.
-
-    Falls back to the type name for callables such as `functools.partial`
-    that have no `__qualname__`.
-    """
-    return getattr(fn, "__qualname__", type(fn).__name__)
-
-
-def linked_root_span_kwargs(context=None):
-    """
-    Keyword arguments that start a new root span with a ``Link`` back to
-    the current span.
-
-    Use this for work that can outlive the span that caused it, such as a
-    background task or a ``block=False`` write.
-
-    Pass ``context`` to link to the span in a previously captured context
-    instead of the current one. If there is no valid span, no link is added.
-
-    Works with any tracer::
-
-        with my_tracer.start_as_current_span(
-            "myplugin.job", **linked_root_span_kwargs()
-        ):
-            ...
-    """
-    cause = get_current_span(context).get_span_context()
-    links = [Link(cause)] if cause.is_valid else []
-    return {"context": otel_context_api.Context(), "links": links}
-
-
-# Keywords that can be recorded as db.operation.name. SQL can be supplied by
-# users, so an allowlist keeps the number of distinct values small.
-DB_OPERATION_ALLOWLIST = frozenset(
-    {
-        "SELECT",
-        "INSERT",
-        "UPDATE",
-        "DELETE",
-        "CREATE",
-        "DROP",
-        "ALTER",
-        "PRAGMA",
-        "EXPLAIN",
-        "REPLACE",
-        "VACUUM",
-        "ANALYZE",
-        "WITH",
-    }
-)
-
-_LEADING_KEYWORD = re.compile(r"^\s*([A-Za-z]+)")
-
-
-def sql_operation_name(sql: str) -> str | None:
-    """
-    The statement's leading keyword if it is in the allowlist, else None.
-
-    Statements that start with a comment or "(" return None. Statements
-    starting with a CTE return `WITH`. Only call this for a single statement.
-    """
-    match = _LEADING_KEYWORD.match(sql)
-    if not match:
-        return None
-    keyword = match.group(1).upper()
-    if keyword in DB_OPERATION_ALLOWLIST:
-        return keyword
-    return None
-
-
-# --- The HTTP request span ------------------------------------------------
-
-
-class _ScopeHeadersGetter(Getter):
-    "Read W3C trace context from an ASGI scope's headers."
-
-    def get(self, carrier, key):
-        wanted = key.lower().encode("latin-1")
-        values = [v.decode("latin-1") for k, v in carrier if k.lower() == wanted]
-        return values or None
-
-    def keys(self, carrier):
-        return [k.decode("latin-1") for k, _ in carrier]
-
-
-_HEADERS_GETTER = _ScopeHeadersGetter()
-
-
-# Methods defined by RFC 9110 plus PATCH (RFC 5789). Anything else is
-# recorded as `_OTHER`, as recommended by semantic conventions.
-_KNOWN_METHODS = frozenset(
-    {"GET", "HEAD", "POST", "PUT", "DELETE", "CONNECT", "OPTIONS", "TRACE", "PATCH"}
-)
-
-
-def clamp_http_method(method):
-    "The request method if it is one we recognise, else ``_OTHER``."
-    method = (method or "").upper()
-    return method if method in _KNOWN_METHODS else "_OTHER"
-
-
-def _first_header(headers, name):
-    "The first value of a header, decoded, or None."
-    for key, value in headers:
-        if key.lower() == name:
-            return value.decode("latin-1")
-    return None
-
-
-def _url_path(scope):
-    """
-    The request path, with any query string removed.
-
-    Prefers `raw_path`, which preserves encoded slashes in database and
-    table names. Some clients include the query string in `raw_path`, so
-    that is stripped as well.
-    """
-    raw_path = scope.get("raw_path")
-    if raw_path:
-        if isinstance(raw_path, bytes):
-            raw_path = raw_path.decode("latin-1")
-        return raw_path.split("?", 1)[0]
-    return scope.get("path", "")
-
-
-# The request span is passed to the router in the ASGI scope, because a
-# plugin's asgi_wrapper() middleware may have made its own span current.
-# Absent if the span is not recording.
-REQUEST_SPAN_SCOPE_KEY = "datasette.telemetry.request_span"
-
-
-def request_span(scope):
-    """
-    The recording request span for an ASGI scope, or None.
-
-    Falls back to the current span, for when Datasette is running under
-    other instrumentation.
-    """
-    span = scope.get(REQUEST_SPAN_SCOPE_KEY)
-    if span is None:
-        span = otel_trace.get_current_span()
-    return span if span.is_recording() else None
-
-
-class TelemetryMiddleware:
-    """
-    One `SpanKind.SERVER` span per HTTP request.
-
-    The span ends after the full response, including any streamed body,
-    has been sent.
-    """
-
-    def __init__(self, app):
-        self.app = app
-
-    async def __call__(self, scope, receive, send):
-        # Pass lifespan and websocket scopes straight through
-        if scope["type"] != "http":
-            await self.app(scope, receive, send)
-            return
-        headers = scope.get("headers") or []
-        # Uses the global propagator, configured with OTEL_PROPAGATORS
-        context = extract(headers, getter=_HEADERS_GETTER)
-        method = clamp_http_method(scope.get("method", ""))
-        # Renamed to include the route once routing has happened
-        with tracer.start_as_current_span(
-            method, context=context, kind=SpanKind.SERVER
-        ) as span:
-            if not span.is_recording():
-                # No provider installed, or the trace was not sampled
-                await self.app(scope, receive, send)
-                return
-            span.set_attribute(HTTP_REQUEST_METHOD, method)
-            span.set_attribute(URL_PATH, _url_path(scope))
-            scheme = scope.get("scheme")
-            if scheme:
-                span.set_attribute(URL_SCHEME, scheme)
-            host = _first_header(headers, b"host")
-            if host:
-                span.set_attribute(SERVER_ADDRESS, host)
-            user_agent = _first_header(headers, b"user-agent")
-            if user_agent:
-                span.set_attribute(USER_AGENT_ORIGINAL, user_agent)
-            if _in_datasette_client.get():
-                span.set_attribute(INTERNAL_CLIENT, True)
-
-            scope = dict(scope, **{REQUEST_SPAN_SCOPE_KEY: span})
-
-            # Some responses are sent without a Response object, so the
-            # status is captured by wrapping send()
-            status_holder = {}
-
-            async def wrapped_send(message):
-                if (
-                    message["type"] == "http.response.start"
-                    and "status" not in status_holder
-                ):
-                    status_holder["status"] = message["status"]
-                await send(message)
-
-            escaped = False
-            try:
-                await self.app(scope, receive, wrapped_send)
-            except BaseException as exception:
-                # Includes asyncio.CancelledError when a client disconnects
-                escaped = True
-                span.set_attribute(ERROR_TYPE, type(exception).__name__)
-                span.set_status(Status(StatusCode.ERROR, str(exception)))
-                raise
-            finally:
-                status = status_holder.get("status")
-                if status is not None:
-                    span.set_attribute(HTTP_RESPONSE_STATUS_CODE, status)
-                    # 4xx responses are not errors for a server span. If an
-                    # exception escaped, keep its class name as error.type.
-                    if status >= 500 and not escaped:
-                        span.set_status(Status(StatusCode.ERROR))
-                        span.set_attribute(ERROR_TYPE, str(status))
-
-
-# --- Metrics --------------------------------------------------------------
-
-
-def _duration_attributes(database_name, operation):
-    return {
-        DB_SYSTEM: "sqlite",
-        DB_NAMESPACE: database_name,
-        OPERATION: operation,
-    }
-
-
-# Instruments use plain text descriptions. The registry entries have longer
-# reStructuredText descriptions for the documentation.
-
-sql_operation_duration = meter.create_histogram(
-    M_OPERATION_DURATION,
-    unit=M_OPERATION_DURATION.unit,
-    description="Duration of a SQL operation issued by Datasette",
-    explicit_bucket_boundaries_advisory=M_OPERATION_DURATION.buckets,
-)
-
-write_queue_wait = meter.create_histogram(
-    M_WRITE_QUEUE_WAIT,
-    unit=M_WRITE_QUEUE_WAIT.unit,
-    description=(
-        "Time a write spent queued behind the single write thread for its database"
-    ),
-    explicit_bucket_boundaries_advisory=M_WRITE_QUEUE_WAIT.buckets,
-)
-
-queries_interrupted = meter.create_counter(
-    M_QUERIES_INTERRUPTED,
-    unit=M_QUERIES_INTERRUPTED.unit,
-    description="Queries cancelled for exceeding sql_time_limit_ms",
-)
-
-
-@contextmanager
-def record_operation_duration(database_name, operation):
-    """
-    Record `db.client.operation.duration` for one SQL operation.
-
-    Sets `error.type` to the exception class on failure. For a `block=False`
-    write this measures the time taken to enqueue the write.
-    """
-    attributes = _duration_attributes(database_name, operation)
-    started = time.perf_counter()
-    try:
-        yield
-    except BaseException as exception:
-        attributes[ERROR_TYPE] = type(exception).__qualname__
-        raise
-    finally:
-        sql_operation_duration.record(time.perf_counter() - started, attributes)
-
-
-def record_write_queue_wait(database_name, waited_ns):
-    write_queue_wait.record(waited_ns / 1e9, {DB_NAMESPACE: database_name})
-
-
-def record_query_interrupted(database_name):
-    queries_interrupted.add(1, {DB_NAMESPACE: database_name})
-
-
-# Live Datasette instances reported by the gauges below. The lock is needed
-# because gauge callbacks run on the SDK's collection thread.
-#
-# The pool gauges do not identify which instance they came from, so they
-# are only meaningful for a process running a single Datasette instance.
-_live_datasettes = weakref.WeakSet()
-_live_datasettes_lock = threading.Lock()
-
-
-def register_datasette(ds):
-    "Start reporting pool/queue gauges for this Datasette instance."
-    with _live_datasettes_lock:
-        _live_datasettes.add(ds)
-
-
-def unregister_datasette(ds):
-    "Stop reporting gauges for an instance that has been closed."
-    with _live_datasettes_lock:
-        _live_datasettes.discard(ds)
-
-
-def _live_instances():
-    with _live_datasettes_lock:
-        return list(_live_datasettes)
-
-
-def _databases_of(ds):
-    "Every Database attached to an instance, including the internal database."
-    databases = list(ds.databases.values())
-    internal = getattr(ds, "_internal_database", None)
-    if internal is not None:
-        databases.append(internal)
-    return databases
-
-
-def observe_sql_thread_limit(options=None):
-    "Size of the shared read-query thread pool (the num_sql_threads setting)."
-    for ds in _live_instances():
-        if ds.executor is None:
-            # num_sql_threads=0 - queries run on the event loop, no pool.
-            continue
-        yield otel_metrics.Observation(ds.setting("num_sql_threads"), {})
-
-
-def observe_sql_thread_queue_depth(options=None):
-    """
-    Read queries waiting for a free thread in the shared pool.
-
-    `_work_queue` is a private attribute of ThreadPoolExecutor, so this
-    reports nothing if it is missing.
-    """
-    for ds in _live_instances():
-        if ds.executor is None:
-            continue
-        work_queue = getattr(ds.executor, "_work_queue", None)
-        if work_queue is None:
-            continue
-        yield otel_metrics.Observation(work_queue.qsize(), {})
-
-
-def observe_pending_queries(options=None):
-    """
-    Read queries submitted to the pool and not yet finished, per database.
-
-    Reads `len()` without `_pending_execute_futures_lock` to avoid blocking
-    queries.
-    """
-    for ds in _live_instances():
-        for db in _databases_of(ds):
-            yield otel_metrics.Observation(
-                len(db._pending_execute_futures), {DB_NAMESPACE: db.name}
-            )
-
-
-def observe_write_queue_depth(options=None):
-    "Writes queued behind the single write thread, per database."
-    for ds in _live_instances():
-        for db in _databases_of(ds):
-            write_queue = db._write_queue
-            if write_queue is None:
-                # No write has ever been queued for this database.
-                continue
-            yield otel_metrics.Observation(write_queue.qsize(), {DB_NAMESPACE: db.name})
-
-
-def observe_open_connections(options=None):
-    "Open SQLite connections tracked for closing, per database."
-    for ds in _live_instances():
-        for db in _databases_of(ds):
-            yield otel_metrics.Observation(
-                len(db._all_connections), {DB_NAMESPACE: db.name}
-            )
-
-
-sql_thread_limit_gauge = meter.create_observable_gauge(
-    M_THREADS_LIMIT,
-    callbacks=[observe_sql_thread_limit],
-    unit=M_THREADS_LIMIT.unit,
-    description="Maximum concurrent read queries (the num_sql_threads setting)",
-)
-
-sql_thread_queue_depth_gauge = meter.create_observable_gauge(
-    M_THREADS_QUEUE_DEPTH,
-    callbacks=[observe_sql_thread_queue_depth],
-    unit=M_THREADS_QUEUE_DEPTH.unit,
-    description="Read queries waiting for a free thread in the shared SQL pool",
-)
-
-pending_queries_gauge = meter.create_observable_gauge(
-    M_QUERIES_PENDING,
-    callbacks=[observe_pending_queries],
-    unit=M_QUERIES_PENDING.unit,
-    description="Read queries submitted to the pool and not yet complete",
-)
-
-write_queue_depth_gauge = meter.create_observable_gauge(
-    M_WRITE_QUEUE_DEPTH,
-    callbacks=[observe_write_queue_depth],
-    unit=M_WRITE_QUEUE_DEPTH.unit,
-    description="Writes queued behind a database's single write thread",
-)
-
-open_connections_gauge = meter.create_observable_gauge(
-    M_CONNECTIONS_OPEN,
-    callbacks=[observe_open_connections],
-    unit=M_CONNECTIONS_OPEN.unit,
-    description="Open SQLite connections tracked for closing",
-)

datasette/telemetry_registry.py

@@ -1,502 +0,0 @@
-"""
-Every span, metric and attribute that Datasette emits.
-
-These entries are used by the instrumentation code, by `docs/telemetry_doc.py`
-to generate the documentation, and by `tests/test_telemetry_registry.py` to
-check that the emitted telemetry matches the registry.
-"""
-
-from opentelemetry.trace import SpanKind
-
-
-class Attribute(str):
-    """
-    A span attribute key, carrying its own documentation.
-
-    Subclasses `str` so it can be handed straight to `set_attribute()`.
-
-    Part of Datasette's public plugin API - plugins declare their own
-    telemetry registries with these classes. See the "Telemetry for plugin
-    authors" documentation.
-    """
-
-    __slots__ = ("description", "optional", "values")
-
-    def __new__(cls, name, description, optional=False, values=None):
-        self = super().__new__(cls, name)
-        self.description = description
-        self.optional = optional
-        # The allowed values for this attribute, or None to allow any value
-        self.values = frozenset(values) if values is not None else None
-        return self
-
-    def __reduce__(self):
-        # Copies and pickles become a plain str, since __new__ requires the
-        # extra arguments. ConsoleMetricExporter deepcopies attribute keys.
-        return (str, (str(self),))
-
-    def __repr__(self):
-        return f"Attribute({str(self)!r})"
-
-
-class SpanName(str):
-    """A span name, carrying its documentation and the attributes it may set.
-
-    Part of Datasette's public plugin API, like `Attribute`.
-    """
-
-    __slots__ = ("attributes", "description", "dynamic", "kind", "prefix")
-
-    def __new__(
-        cls,
-        name,
-        description,
-        attributes=(),
-        prefix=False,
-        dynamic=False,
-        kind=SpanKind.INTERNAL,
-    ):
-        self = super().__new__(cls, name)
-        self.description = description
-        self.attributes = tuple(attributes)
-        # Match emitted names that start with this prefix, for names with a
-        # variable suffix such as SpanName("chat ", ..., prefix=True)
-        self.prefix = prefix
-        # The emitted name is built at runtime, so `span_for()` matches it by
-        # span kind. The entry's string is a template for the documentation.
-        self.dynamic = dynamic
-        self.kind = kind
-        return self
-
-    def __reduce__(self):
-        # See Attribute.__reduce__.
-        return (str, (str(self),))
-
-    def __repr__(self):
-        return f"SpanName({str(self)!r})"
-
-
-class MetricName(str):
-    "A metric name, carrying its instrument kind, unit and attributes."
-
-    __slots__ = ("attributes", "buckets", "description", "kind", "unit")
-
-    def __new__(cls, name, kind, unit, description, attributes=(), buckets=None):
-        self = super().__new__(cls, name)
-        self.kind = kind
-        self.unit = unit
-        self.description = description
-        self.attributes = tuple(attributes)
-        # Explicit bucket boundaries, for histograms only
-        self.buckets = tuple(buckets) if buckets is not None else None
-        return self
-
-    def __reduce__(self):
-        # See Attribute.__reduce__.
-        return (str, (str(self),))
-
-    def __repr__(self):
-        return f"MetricName({str(self)!r})"
-
-
-COUNTER = "Counter"
-UPDOWN_COUNTER = "UpDownCounter"
-HISTOGRAM = "Histogram"
-GAUGE = "Observable gauge"
-
-
-# --- Attributes -----------------------------------------------------------
-
-HTTP_REQUEST_METHOD = Attribute(
-    "http.request.method",
-    "The HTTP request method. Methods outside the nine defined by RFC 9110 "
-    "and RFC 5789 are recorded as ``_OTHER``.",
-)
-HTTP_RESPONSE_STATUS_CODE = Attribute(
-    "http.response.status_code",
-    "The HTTP response status code. Omitted if no response was started.",
-    optional=True,
-)
-HTTP_ROUTE = Attribute(
-    "http.route",
-    "The regular expression for the matched route, for example "
-    "``/(?P<database>[^\\/\\.]+)/(?P<table>[^\\/\\.]+)(\\.(?P<format>\\w+))?$`` "
-    "for a table page. Use this attribute to group requests by route. "
-    "Omitted when no route matches.",
-    optional=True,
-)
-URL_PATH = Attribute(
-    "url.path",
-    "The URL path, excluding the query string.",
-)
-URL_SCHEME = Attribute("url.scheme", "``http`` or ``https``.")
-SERVER_ADDRESS = Attribute(
-    "server.address",
-    "The ``Host`` header, including any ``:port`` suffix. This value is "
-    "supplied by the client.",
-    optional=True,
-)
-USER_AGENT_ORIGINAL = Attribute(
-    "user_agent.original",
-    "The ``User-Agent`` header, verbatim. Omitted if the client sent none.",
-    optional=True,
-)
-INTERNAL_CLIENT = Attribute(
-    "datasette.internal_client",
-    "``True`` for requests made through ``datasette.client``. Calls made "
-    "inside another request produce a nested ``SERVER`` span. Filter on "
-    "this attribute to exclude internal requests from request counts. "
-    "Omitted for requests received over the network.",
-    optional=True,
-)
-ERROR_TYPE = Attribute(
-    "error.type",
-    "The exception class name for a failed operation. On HTTP spans, also "
-    "set to the status code as a string for 5xx responses. A 4xx response "
-    "alone does not set this attribute or an error status.",
-    optional=True,
-)
-
-DB_SYSTEM = Attribute("db.system", "Always ``sqlite``.")
-DB_NAMESPACE = Attribute("db.namespace", "Name of the database being queried.")
-OPERATION = Attribute(
-    "datasette.operation",
-    "Whether the operation was a read or a write.",
-    values={"read", "write"},
-)
-DB_QUERY_TEXT = Attribute(
-    "db.query.text",
-    "The SQL, truncated to 2048 characters. Bound parameter values are not "
-    "recorded. For callback methods, ``datasette.callback`` is recorded instead.",
-    optional=True,
-)
-CALLBACK = Attribute(
-    "datasette.callback",
-    "The qualified name of the Python callable passed to ``execute_fn()``, "
-    "``execute_write_fn()`` or ``execute_isolated_fn()``, for example "
-    "``TableInsertView.post.<locals>.insert_or_upsert_rows``. Set instead of "
-    "``db.query.text``. Lambdas appear as ``<lambda>``; use a named function "
-    "for a more descriptive span.",
-    optional=True,
-)
-DB_OPERATION_NAME = Attribute(
-    "db.operation.name",
-    "The statement's leading keyword, such as ``SELECT``, ``INSERT`` or "
-    "``CREATE``, if it matches the supported allowlist. Statements beginning "
-    "with a common table expression report ``WITH``. Omitted for unrecognized "
-    "keywords and ``execute_write_script()``.",
-    optional=True,
-)
-PARAM_COUNT = Attribute(
-    "datasette.param_count",
-    "Number of bound parameters. Recorded instead of the values themselves.",
-    optional=True,
-)
-PARAM_SETS = Attribute(
-    "datasette.param_sets",
-    "Number of parameter sets consumed by ``execute_write_many()``. "
-    "The parameter values are not recorded.",
-    optional=True,
-)
-TIME_LIMIT_MS = Attribute(
-    "datasette.time_limit_ms",
-    "Time limit applied to the read query, in milliseconds: "
-    ":ref:`setting_sql_time_limit_ms` or a shorter ``custom_time_limit``.",
-    optional=True,
-)
-ROWS_RETURNED = Attribute(
-    "datasette.rows_returned",
-    "Number of rows returned by a successful read query.",
-    optional=True,
-)
-TRUNCATED = Attribute(
-    "datasette.truncated",
-    "True if the result was cut short by :ref:`setting_max_returned_rows`.",
-    optional=True,
-)
-INTERRUPTED = Attribute(
-    "datasette.interrupted",
-    "True if the query exceeded its time limit. The span status is set to "
-    "``ERROR`` unless the caller used a ``custom_time_limit`` shorter than "
-    ":ref:`setting_sql_time_limit_ms`, in which case the status is left unset.",
-    optional=True,
-)
-SQL_ERROR_SUPPRESSED = Attribute(
-    "datasette.sql_error_suppressed",
-    "True for a non-timeout SQL error with ``log_sql_errors=False``. The "
-    "exception is still raised, but the span status is left unset.",
-    optional=True,
-)
-EXECUTESCRIPT = Attribute(
-    "datasette.executescript",
-    "True for ``execute_write_script()``, which runs multiple statements.",
-    optional=True,
-)
-EXECUTEMANY = Attribute(
-    "datasette.executemany",
-    "True for ``execute_write_many()``, which runs one statement against many "
-    "parameter sets.",
-    optional=True,
-)
-ISOLATED_CONNECTION = Attribute(
-    "datasette.isolated_connection",
-    "True if the write ran on its own connection rather than the shared write "
-    "connection.",
-)
-TRANSACTION = Attribute(
-    "datasette.transaction",
-    "False for statements such as ``VACUUM`` that cannot run inside a transaction.",
-)
-
-
-# --- Spans ----------------------------------------------------------------
-
-HTTP_REQUEST = SpanName(
-    "{http.request.method} {http.route}",
-    "One span per HTTP request, containing spans from plugin middleware and "
-    "database operations. Named for the HTTP method and matched route, or "
-    "just the method if no route matches. Incoming ``traceparent`` headers "
-    "are extracted using the global propagator to continue the caller's "
-    "trace. Incoming ``baggage`` is not propagated into plugin or downstream "
-    "context in this release. Set ``OTEL_PROPAGATORS=none`` to disable "
-    "extraction. For public instances, strip trace context headers at your "
-    "proxy if callers should not supply trace context.",
-    (
-        HTTP_REQUEST_METHOD,
-        HTTP_ROUTE,
-        URL_PATH,
-        URL_SCHEME,
-        SERVER_ADDRESS,
-        USER_AGENT_ORIGINAL,
-        HTTP_RESPONSE_STATUS_CODE,
-        ERROR_TYPE,
-        INTERNAL_CLIENT,
-    ),
-    dynamic=True,
-    kind=SpanKind.SERVER,
-)
-
-DB_QUERY = SpanName(
-    "db.query",
-    "A SQL operation, including time spent queued for a worker thread. For "
-    "``block=False`` writes, the span ends after the write is queued. "
-    "Callback methods record ``datasette.callback`` in place of ``db.query.text``.",
-    (
-        DB_SYSTEM,
-        DB_NAMESPACE,
-        DB_QUERY_TEXT,
-        CALLBACK,
-        DB_OPERATION_NAME,
-        PARAM_COUNT,
-        PARAM_SETS,
-        TIME_LIMIT_MS,
-        ROWS_RETURNED,
-        TRUNCATED,
-        INTERRUPTED,
-        SQL_ERROR_SUPPRESSED,
-        EXECUTESCRIPT,
-        EXECUTEMANY,
-    ),
-    kind=SpanKind.CLIENT,
-)
-
-DB_QUERY_EXECUTE = SpanName(
-    "db.query.execute",
-    "The read executing inside a SQL worker thread. Child of ``db.query``; the "
-    "gap between the two is time spent waiting for a thread.",
-)
-
-DB_WRITE_QUEUE_WAIT = SpanName(
-    "db.write.queue_wait",
-    "Time a write spent waiting in its database's write queue. For "
-    "``block=True``, this is a child of ``db.query``. For ``block=False``, "
-    "it is a root span linked to the span that queued the write, since the "
-    "write can outlive that request.",
-)
-
-DB_WRITE_EXECUTE = SpanName(
-    "db.write.execute",
-    "The write executing on the write thread. For ``block=True``, this is "
-    "a child of ``db.query``. For ``block=False``, it is a root span linked "
-    "to the span that queued the write.",
-    (ISOLATED_CONNECTION, TRANSACTION),
-)
-
-STARTUP = SpanName(
-    "datasette.startup",
-    "Startup work performed by ``invoke_startup()``, including registration "
-    "hooks, schema catalog updates, saved queries, column type configuration "
-    "and the ``startup`` hook. Runs during instance startup, either before "
-    "serving requests or as part of the first request.",
-)
-
-SPANS = (
-    HTTP_REQUEST,
-    DB_QUERY,
-    DB_QUERY_EXECUTE,
-    DB_WRITE_QUEUE_WAIT,
-    DB_WRITE_EXECUTE,
-    STARTUP,
-)
-
-
-def span_for(emitted_name, kind=None, spans=None):
-    """
-    Resolve an emitted span name to its registry entry, or None.
-
-    Exact matches take precedence over `prefix=True` entries, which take
-    precedence over `dynamic=True` entries matched by `kind`.
-
-    `spans` defaults to Datasette's own registry.
-    """
-    if spans is None:
-        spans = SPANS
-    for span in spans:
-        if span.dynamic:
-            continue
-        if emitted_name == span:
-            return span
-    for span in spans:
-        if span.prefix and emitted_name.startswith(span):
-            return span
-    if kind is not None:
-        for span in spans:
-            if span.dynamic and span.kind == kind:
-                return span
-    return None
-
-
-def metric_for(emitted_name, metrics=None):
-    """
-    Resolve an emitted metric name to its registry entry, or None.
-
-    `metrics` defaults to Datasette's own registry.
-    """
-    if metrics is None:
-        metrics = METRICS
-    for metric in metrics:
-        if emitted_name == metric:
-            return metric
-    return None
-
-
-def attribute_allowed(entry, emitted_key):
-    """
-    Whether `emitted_key` is a registered attribute of `entry`.
-
-    `entry` is a `SpanName` or a `MetricName` - both carry `.attributes`.
-    """
-    if entry is None:
-        return False
-    return emitted_key in entry.attributes
-
-
-def attribute_value_allowed(entry, emitted_key, value):
-    """
-    Whether `value` is permitted for `emitted_key` on `entry` (a `SpanName`
-    or a `MetricName`).
-
-    Any value is allowed if the attribute does not declare `values=`.
-    """
-    if entry is None:
-        return False
-    for attribute in entry.attributes:
-        if attribute == emitted_key:
-            return attribute.values is None or value in attribute.values
-    return False
-
-
-# --- Metrics --------------------------------------------------------------
-
-# Bucket boundaries in seconds for every duration histogram. OpenTelemetry's
-# defaults are designed for milliseconds and would put almost every SQLite
-# query in the first bucket. These are the semantic conventions' recommended
-# boundaries for db.client.operation.duration, plus 0.0001 and 0.0005 for
-# fast in-process SQLite queries.
-DURATION_BUCKETS = (0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10)
-
-M_OPERATION_DURATION = MetricName(
-    "db.client.operation.duration",
-    HISTOGRAM,
-    "s",
-    "Duration of a SQL operation, including callback-based calls such as "
-    "``execute_fn()``. For ``block=False`` writes, measures enqueue time.",
-    (DB_SYSTEM, DB_NAMESPACE, OPERATION, ERROR_TYPE),
-    buckets=DURATION_BUCKETS,
-)
-
-M_WRITE_QUEUE_WAIT = MetricName(
-    "datasette.write.queue_wait",
-    HISTOGRAM,
-    "s",
-    "Time each write waited in its database's write queue.",
-    (DB_NAMESPACE,),
-    buckets=DURATION_BUCKETS,
-)
-
-M_QUERIES_INTERRUPTED = MetricName(
-    "datasette.sql.queries.interrupted",
-    COUNTER,
-    "{query}",
-    "Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. A "
-    "rising rate can indicate that queries need optimization or a higher "
-    "time limit. Caller-selected timeouts shorter than this limit, such as "
-    "those used for facet suggestion, are excluded.",
-    (DB_NAMESPACE,),
-)
-
-M_THREADS_LIMIT = MetricName(
-    "datasette.sql.threads.limit",
-    GAUGE,
-    "{thread}",
-    "Maximum concurrent read queries, configured by "
-    ":ref:`setting_num_sql_threads`. Not reported when ``num_sql_threads`` "
-    "is ``0``.",
-)
-
-M_THREADS_QUEUE_DEPTH = MetricName(
-    "datasette.sql.threads.queue_depth",
-    GAUGE,
-    "{query}",
-    "Read queries waiting for a free SQL thread. Sustained values above "
-    "zero indicate a saturated read pool.",
-)
-
-M_QUERIES_PENDING = MetricName(
-    "datasette.sql.queries.pending",
-    GAUGE,
-    "{query}",
-    "Read queries submitted to the pool and not yet complete. Sum across "
-    "databases and compare with ``datasette.sql.threads.limit`` to assess "
-    "pool usage.",
-    (DB_NAMESPACE,),
-)
-
-M_WRITE_QUEUE_DEPTH = MetricName(
-    "datasette.write.queue_depth",
-    GAUGE,
-    "{write}",
-    "Writes waiting for a database's single write thread. Increasing "
-    "``num_sql_threads`` does not increase write concurrency. Not reported for "
-    "databases that have never been written to.",
-    (DB_NAMESPACE,),
-)
-
-M_CONNECTIONS_OPEN = MetricName(
-    "datasette.connections.open",
-    GAUGE,
-    "{connection}",
-    "Open SQLite connections managed by Datasette.",
-    (DB_NAMESPACE,),
-)
-
-METRICS = (
-    M_OPERATION_DURATION,
-    M_WRITE_QUEUE_WAIT,
-    M_QUERIES_INTERRUPTED,
-    M_THREADS_LIMIT,
-    M_THREADS_QUEUE_DEPTH,
-    M_QUERIES_PENDING,
-    M_WRITE_QUEUE_DEPTH,
-    M_CONNECTIONS_OPEN,
-)

datasette/telemetry_testing.py

@@ -1,427 +0,0 @@
-"""
-Pytest helpers for testing OpenTelemetry instrumentation - Datasette's own
-and any plugin's. Part of Datasette's public plugin API; see the "Telemetry
-for plugin authors" documentation.
-
-Usage from a plugin's ``conftest.py``::
-
-    from datasette.telemetry_testing import (  # noqa: F401
-        MetricsCollector,
-        otel_metrics,
-        otel_meter_provider,
-        otel_provider,
-        otel_spans,
-    )
-
-Tests can then use the ``otel_spans`` and ``otel_metrics`` fixtures. The
-OpenTelemetry SDK is imported lazily, and the fixtures skip if it is not
-installed.
-"""
-
-import subprocess
-import sys
-
-import pytest
-
-from .telemetry_registry import (
-    attribute_allowed,
-    attribute_value_allowed,
-    metric_for,
-    span_for,
-)
-
-_span_exporter = None
-_metric_reader = None
-
-
-def install_span_exporter():
-    """
-    Install a TracerProvider + InMemorySpanExporter once per process and
-    return the exporter, or None when the SDK is not installed.
-
-    Uses `SimpleSpanProcessor` so spans are exported as soon as they end.
-    """
-    global _span_exporter
-    if _span_exporter is not None:
-        return _span_exporter
-    try:
-        from opentelemetry import trace as otel_trace
-        from opentelemetry.sdk.trace import TracerProvider
-        from opentelemetry.sdk.trace.export import SimpleSpanProcessor
-        from opentelemetry.sdk.trace.export.in_memory_span_exporter import (
-            InMemorySpanExporter,
-        )
-    except ImportError:
-        return None
-    exporter = InMemorySpanExporter()
-    provider = TracerProvider()
-    provider.add_span_processor(SimpleSpanProcessor(exporter))
-    otel_trace.set_tracer_provider(provider)
-    # set_tracer_provider() is ignored if a provider was already installed,
-    # in which case the fixtures skip
-    if otel_trace.get_tracer_provider() is not provider:
-        return None
-    _span_exporter = exporter
-    return exporter
-
-
-def install_metric_reader():
-    """
-    Install a MeterProvider + InMemoryMetricReader once per process and
-    return the reader, or None when the SDK is not installed.
-
-    Uses delta temporality for counters and histograms, so each collection
-    only reports measurements since the previous one.
-    """
-    global _metric_reader
-    if _metric_reader is not None:
-        return _metric_reader
-    try:
-        from opentelemetry import metrics as otel_metrics_api
-        from opentelemetry.sdk.metrics import Counter, Histogram, MeterProvider
-        from opentelemetry.sdk.metrics.export import (
-            AggregationTemporality,
-            InMemoryMetricReader,
-        )
-    except ImportError:
-        return None
-    reader = InMemoryMetricReader(
-        preferred_temporality={
-            Counter: AggregationTemporality.DELTA,
-            Histogram: AggregationTemporality.DELTA,
-        }
-    )
-    provider = MeterProvider(metric_readers=[reader])
-    otel_metrics_api.set_meter_provider(provider)
-    if otel_metrics_api.get_meter_provider() is not provider:
-        return None
-    _metric_reader = reader
-    return reader
-
-
-@pytest.fixture(scope="session", autouse=True)
-def otel_provider():
-    "Install the span exporter once per test session, before any spans are created."
-    install_span_exporter()
-
-
-@pytest.fixture(scope="session", autouse=True)
-def otel_meter_provider():
-    "Install the metric reader once per test session."
-    install_metric_reader()
-
-
-@pytest.fixture(autouse=True)
-def otel_reset():
-    "Clear recorded spans and drain collected metrics after every test."
-    yield
-    if _span_exporter is not None:
-        _span_exporter.clear()
-    if _metric_reader is not None:
-        _metric_reader.get_metrics_data()
-
-
-@pytest.fixture
-def otel_spans():
-    """
-    The in-memory span exporter, cleared before the test. Call
-    `.get_finished_spans()` to retrieve spans.
-    """
-    pytest.importorskip("opentelemetry.sdk")
-    exporter = install_span_exporter()
-    if exporter is None:
-        pytest.skip("OpenTelemetry SDK provider was not installed")
-    exporter.clear()
-    yield exporter
-
-
-class MetricsCollector:
-    """
-    Wraps an `InMemoryMetricReader`.
-
-    `collect()` runs a collection cycle and stores a snapshot, which
-    `points()` and `point()` then query.
-    """
-
-    def __init__(self, reader):
-        self.reader = reader
-        self.snapshot = {}
-        # (instrumentation scope name, sdk Metric) pairs from the last collect()
-        self.collected = []
-
-    def collect(self):
-        self.snapshot = {}
-        self.collected = []
-        data = self.reader.get_metrics_data()
-        if data is None:
-            return self.snapshot
-        for resource_metrics in data.resource_metrics:
-            for scope_metrics in resource_metrics.scope_metrics:
-                scope_name = scope_metrics.scope.name if scope_metrics.scope else None
-                for metric in scope_metrics.metrics:
-                    self.snapshot.setdefault(metric.name, []).extend(
-                        metric.data.data_points
-                    )
-                    self.collected.append((scope_name, metric))
-        return self.snapshot
-
-    def points(self, name, attributes=None):
-        "Data points for `name` whose attributes are a superset of `attributes`."
-        found = []
-        for point in self.snapshot.get(name, []):
-            point_attributes = dict(point.attributes or {})
-            if all(point_attributes.get(k) == v for k, v in (attributes or {}).items()):
-                found.append(point)
-        return found
-
-    def point(self, name, attributes=None):
-        "The single matching data point, asserting there is exactly one."
-        found = self.points(name, attributes)
-        assert len(found) == 1, (
-            f"expected exactly one {name} point matching {attributes}, "
-            f"got {len(found)}: {found}"
-        )
-        return found[0]
-
-
-@pytest.fixture
-def otel_metrics():
-    "A `MetricsCollector`, drained before the test so counts start from zero."
-    pytest.importorskip("opentelemetry.sdk")
-    reader = install_metric_reader()
-    if reader is None:
-        pytest.skip("OpenTelemetry SDK meter provider was not installed")
-    reader.get_metrics_data()
-    yield MetricsCollector(reader)
-
-
-def _scoped(finished_spans, scope_name):
-    if scope_name is None:
-        return list(finished_spans)
-    return [
-        span
-        for span in finished_spans
-        if span.instrumentation_scope and span.instrumentation_scope.name == scope_name
-    ]
-
-
-def assert_spans_conform(registry_spans, finished_spans, scope_name=None):
-    """
-    Assert every finished span is registered in `registry_spans`, sets only
-    registered attributes and uses allowed attribute values.
-
-    Pass `scope_name` to only check spans from that instrumentation scope.
-    """
-    problems = []
-    for span in _scoped(finished_spans, scope_name):
-        entry = span_for(str(span.name), kind=span.kind, spans=registry_spans)
-        if entry is None:
-            problems.append(f"unregistered span: {span.name!r}")
-            continue
-        for key, value in (span.attributes or {}).items():
-            if not attribute_allowed(entry, str(key)):
-                problems.append(f"{span.name}: unregistered attribute {key!r}")
-            elif not attribute_value_allowed(entry, str(key), value):
-                problems.append(
-                    f"{span.name}: {key}={value!r} not in the declared enum"
-                )
-    assert not problems, "\n".join(problems)
-
-
-def assert_spans_covered(registry_spans, finished_spans, scope_name=None):
-    """
-    Assert every entry in `registry_spans` was emitted at least once, with
-    each of its attributes that is not `optional=True`.
-    """
-    spans = _scoped(finished_spans, scope_name)
-    seen_attributes = {}
-    for span in spans:
-        entry = span_for(str(span.name), kind=span.kind, spans=registry_spans)
-        if entry is not None:
-            seen = seen_attributes.setdefault(str(entry), set())
-            seen.update(str(key) for key in (span.attributes or {}))
-    problems = []
-    for entry in registry_spans:
-        if str(entry) not in seen_attributes:
-            problems.append(f"registered span never emitted: {entry!r}")
-            continue
-        required = {
-            str(attribute) for attribute in entry.attributes if not attribute.optional
-        }
-        missing = required - seen_attributes[str(entry)]
-        if missing:
-            problems.append(
-                f"{entry}: registered attributes never emitted: {sorted(missing)}"
-            )
-    assert not problems, "\n".join(problems)
-
-
-# Registry instrument kinds mapped to the SDK data type collected for them.
-# Both counter kinds collect as Sum, distinguished by is_monotonic.
-_KIND_TO_DATA_TYPE = {
-    "Counter": "Sum",
-    "UpDownCounter": "Sum",
-    "Histogram": "Histogram",
-    "Observable gauge": "Gauge",
-}
-_KIND_IS_MONOTONIC = {"Counter": True, "UpDownCounter": False}
-
-
-def _scoped_metrics(collector, scope_name):
-    for scope, metric in collector.collected:
-        if scope_name is None or scope == scope_name:
-            yield metric
-
-
-def assert_metrics_conform(registry_metrics, collector, scope_name=None):
-    """
-    Assert every metric in the collector's last `collect()` is registered in
-    `registry_metrics` with a matching instrument kind and unit, sets only
-    registered attributes and uses allowed attribute values.
-
-    Pass `scope_name` to only check metrics from that instrumentation scope.
-    """
-    problems = set()
-    for metric in _scoped_metrics(collector, scope_name):
-        entry = metric_for(metric.name, metrics=registry_metrics)
-        if entry is None:
-            problems.add(f"unregistered metric: {metric.name!r}")
-            continue
-        expected_data_type = _KIND_TO_DATA_TYPE.get(entry.kind)
-        actual_data_type = type(metric.data).__name__
-        if expected_data_type is not None and actual_data_type != expected_data_type:
-            problems.add(
-                f"{metric.name}: registry declares {entry.kind}, "
-                f"SDK collected {actual_data_type}"
-            )
-        expected_monotonic = _KIND_IS_MONOTONIC.get(entry.kind)
-        actual_monotonic = getattr(metric.data, "is_monotonic", None)
-        if (
-            expected_monotonic is not None
-            and actual_monotonic is not None
-            and actual_monotonic != expected_monotonic
-        ):
-            problems.add(
-                f"{metric.name}: registry declares {entry.kind}, but the "
-                f"collected Sum is_monotonic={actual_monotonic}"
-            )
-        if (metric.unit or "") != (entry.unit or ""):
-            problems.add(
-                f"{metric.name}: instrument unit {metric.unit!r} != "
-                f"registry unit {entry.unit!r}"
-            )
-        for point in metric.data.data_points:
-            for key, value in dict(point.attributes or {}).items():
-                if not attribute_allowed(entry, str(key)):
-                    problems.add(f"{metric.name}: unregistered attribute {key!r}")
-                elif not attribute_value_allowed(entry, str(key), value):
-                    problems.add(
-                        f"{metric.name}: {key}={value!r} not in the declared enum"
-                    )
-    assert not problems, "\n".join(sorted(problems))
-
-
-def assert_metrics_covered(registry_metrics, collector, scope_name=None):
-    """
-    Assert every entry in `registry_metrics` was collected at least once,
-    with each of its attributes that is not `optional=True`.
-
-    Call `collect()` once after the workload and before this check.
-    """
-    seen_attributes = {}
-    for metric in _scoped_metrics(collector, scope_name):
-        entry = metric_for(metric.name, metrics=registry_metrics)
-        if entry is None:
-            continue
-        seen = seen_attributes.setdefault(str(entry), set())
-        for point in metric.data.data_points:
-            seen.update(str(key) for key in dict(point.attributes or {}))
-    problems = []
-    for entry in registry_metrics:
-        if str(entry) not in seen_attributes:
-            problems.append(f"registered metric never collected: {entry!r}")
-            continue
-        required = {
-            str(attribute) for attribute in entry.attributes if not attribute.optional
-        }
-        missing = required - seen_attributes[str(entry)]
-        if missing:
-            problems.append(
-                f"{entry}: registered attributes never collected: {sorted(missing)}"
-            )
-    assert not problems, "\n".join(problems)
-
-
-def assert_no_forbidden_values(
-    forbidden, finished_spans=None, collector=None, scope_name=None
-):
-    """
-    Assert that none of the `forbidden` strings appear anywhere in the
-    emitted telemetry: span names, span attribute values, span event names
-    and attributes, span status descriptions, or metric point attributes.
-
-    Use fake private values such as tokens or email addresses in your test
-    workload, then check that they were not recorded:
-
-        FORBIDDEN = {"secret-token-123", "alice@example.com"}
-        run_workload_using_those_values()
-        assert_no_forbidden_values(
-            FORBIDDEN,
-            finished_spans=otel_spans.get_finished_spans(),
-            collector=otel_metrics,
-        )
-
-    Matches substrings of each value's string form. Empty strings in
-    `forbidden` are ignored. Leave `scope_name` unset to also check
-    Datasette's own telemetry.
-    """
-    needles = [needle for needle in forbidden if needle]
-    leaks = set()
-
-    def check(value, where):
-        text = str(value)
-        for needle in needles:
-            if needle in text:
-                leaks.add(f"{where} contains {needle!r}")
-
-    if finished_spans is not None:
-        for span in _scoped(finished_spans, scope_name):
-            check(span.name, f"span name {str(span.name)!r}")
-            for key, value in (span.attributes or {}).items():
-                check(value, f"{span.name} attribute {key}")
-            for event in span.events or ():
-                check(event.name, f"{span.name} event name")
-                for key, value in (event.attributes or {}).items():
-                    check(value, f"{span.name} event {event.name} attribute {key}")
-            if span.status is not None and span.status.description:
-                check(span.status.description, f"{span.name} status description")
-    if collector is not None:
-        for metric in _scoped_metrics(collector, scope_name):
-            for point in metric.data.data_points:
-                for key, value in dict(point.attributes or {}).items():
-                    check(value, f"metric {metric.name} attribute {key}")
-    assert not leaks, "forbidden values leaked into telemetry:\n" + "\n".join(
-        sorted(leaks)
-    )
-
-
-def assert_package_never_imports_sdk(*module_names):
-    """
-    Import the named modules in a fresh interpreter and assert none of them
-    imported `opentelemetry.sdk`.
-
-    Run the test that calls this early in your suite: on macOS with CPython
-    3.13, starting a subprocess from a process with many threads can crash.
-    """
-    imports = "; ".join(f"import {name}" for name in module_names)
-    code = (
-        f"import sys; {imports}; "
-        "print([m for m in sys.modules if m.startswith('opentelemetry.sdk')])"
-    )
-    result = subprocess.run(
-        [sys.executable, "-c", code], capture_output=True, text=True, check=True
-    )
-    assert result.stdout.strip() == "[]", (
-        f"importing {module_names} pulled in the OpenTelemetry SDK: "
-        f"{result.stdout.strip()}"
-    )

datasette/templates/api_explorer.html

@@ -3,6 +3,7 @@
 {% block title %}API Explorer{% endblock %}
 
 {% block extra_head %}
+<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% endblock %}
 
 {% block content %}
@@ -125,7 +126,7 @@ getForm.addEventListener("submit", (ev) => {
     document.getElementById('response-status').textContent = response.status;
     return response.json();
   }).then((data) => {
-    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
+    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
     errorList.style.display = 'none';
   }).catch((error) => {
     alert(error);
@@ -173,7 +174,7 @@ postForm.addEventListener("submit", (ev) => {
     } else {
       errorList.style.display = 'none';
     }
-    output.querySelector('pre').textContent = JSON.stringify(data, null, 2);
+    output.querySelector('pre').innerHTML = jsonFormatHighlight(data);
     output.style.display = 'block';
   }).catch(err => {
     alert("Error: " + err);

datasette/templates/base.html

@@ -8,7 +8,6 @@
     <link rel="stylesheet" href="{{ url.url }}"{% if url.get("sri") %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}>
 {% endfor %}
 <script>window.datasetteVersion = '{{ datasette_version }}';</script>
-<script src="{{ static('modal.js') }}" defer></script>
 <script src="{{ static('datasette-manager.js') }}" defer></script>
 {% for url in extra_js_urls %}
     <script {% if url.module %}type="module" {% endif %}src="{{ url.url }}"{% if url.get("sri") %} integrity="{{ url.sri }}" crossorigin="anonymous"{% endif %}></script>

datasette/templates/debug_allowed.html

@@ -3,6 +3,7 @@
 {% block title %}Allowed Resources{% endblock %}
 
 {% block extra_head %}
+<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -197,7 +198,7 @@ function displayResults(data) {
     }
 
     // Update raw JSON
-    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
+    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
 }
 
 function displayError(data) {
@@ -207,7 +208,7 @@ function displayError(data) {
 
     resultsContent.innerHTML = `<div class="error-message">Error: ${escapeHtml(data.error || 'Unknown error')}</div>`;
 
-    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
+    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
 }
 
 // Disable child input if parent is empty

datasette/templates/debug_check.html

@@ -3,6 +3,7 @@
 {% block title %}Explain a permission decision{% endblock %}
 
 {% block extra_head %}
+<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 <style>
@@ -237,7 +238,7 @@ function displayResult(data) {
     displayRules(data.explanation);
     displayRestrictions(data.explanation.restrictions);
     displayRequirements(data.explanation.required_actions);
-    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
+    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
 }
 
 function displayRules(explanation) {
@@ -297,7 +298,7 @@ function displayError(data) {
     document.getElementById('matching-rules').innerHTML = '';
     document.getElementById('restrictions-section').style.display = 'none';
     document.getElementById('requirements-section').style.display = 'none';
-    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
+    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
 }
 
 form.addEventListener('submit', event => {

datasette/templates/debug_rules.html

@@ -3,6 +3,7 @@
 {% block title %}Permission Rules{% endblock %}
 
 {% block extra_head %}
+<script src="{{ static('json-format-highlight-1.0.1.js') }}"></script>
 {% include "_permission_ui_styles.html" %}
 {% include "_debug_common_functions.html" %}
 {% endblock %}
@@ -184,7 +185,7 @@ function displayResults(data) {
     }
 
     // Update raw JSON
-    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
+    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
 }
 
 function displayError(data) {
@@ -194,7 +195,7 @@ function displayError(data) {
 
     resultsContent.innerHTML = `<div class="error-message">Error: ${escapeHtml(data.error || 'Unknown error')}</div>`;
 
-    document.getElementById('raw-json').textContent = JSON.stringify(data, null, 2);
+    document.getElementById('raw-json').innerHTML = jsonFormatHighlight(data);
 }
 
 </script>

datasette/templates/index.html

@@ -26,7 +26,8 @@
             {% if database.show_table_row_counts %}{{ "{:,}".format(database.hidden_table_rows_sum) }} rows in {% endif %}{{ database.hidden_tables_count }} hidden table{% if database.hidden_tables_count != 1 %}s{% endif -%}
         {% endif -%}
         {% if database.views_count -%}
-            , {{ "{:,}".format(database.views_count) }} view{% if database.views_count != 1 %}s{% endif %}
+            {% if database.tables_count or database.hidden_tables_count %}, {% endif -%}
+            {{ "{:,}".format(database.views_count) }} view{% if database.views_count != 1 %}s{% endif %}
         {% endif %}
     </p>
     <p>{% for table in database.tables_and_views_truncated %}<a href="{{ urls.table(database.name, table.name) }}"{% if table.count %} title="{{ table.count }} rows"{% endif %}>{{ table.name }}</a>{% if table.private %} 🔒{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}{% if database.tables_and_views_more %}, <a href="{{ urls.database(database.name) }}">...</a>{% endif %}</p>

datasette/templates/table.html

@@ -1,6 +1,6 @@
 {% extends "base.html" %}
 
-{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}{{ "{:,}".format(count - 1) }}+ rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
+{% block title %}{{ database }}: {{ table }}: {% if count_truncated %}>{{ "{:,}".format(count - 1) }} rows{% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}{% if human_description_en %} {{ human_description_en }}{% endif %}{% endblock %}
 
 {% block extra_head %}
 {{- super() -}}
@@ -47,12 +47,11 @@
 {% endif %}
 
 {% if count or human_description_en %}
-    <h3 class="table-summary">
+    <h3>
-        {% if count_truncated %}<span class="table-count" aria-live="polite">{{ "{:,}".format(count - 1) }}+ rows</span>
+        {% if count_truncated %}&gt;{{ "{:,}".format(count - 1) }} rows
-        <button type="button" class="count-all" data-count-url="{{ urls.table(database, table) }}/-/count">count all</button>
+        {% if allow_execute_sql and query.sql %} <a class="count-sql" style="font-size: 0.8em;" href="{{ urls.database_query(database, count_sql) }}">count all</a>{% endif %}
-        <span class="count-error" role="alert"></span>
         {% elif count or count == 0 %}{{ "{:,}".format(count) }} row{% if count == 1 %}{% else %}s{% endif %}{% endif %}
-        {% if human_description_en %}<span class="table-summary-description">{{ human_description_en }}</span>{% endif %}
+        {% if human_description_en %}{{ human_description_en }}{% endif %}
     </h3>
 {% endif %}
 

datasette/utils/__init__.py

@@ -568,7 +568,7 @@ def escape_css_string(s):
 
 
 def escape_sqlite(s):
-    if _boring_keyword_re.fullmatch(s) and (s.lower() not in reserved_words):
+    if _boring_keyword_re.match(s) and (s.lower() not in reserved_words):
         return s
     return '"{}"'.format(s.replace('"', '""'))
 
@@ -820,8 +820,7 @@ def detect_spatialite(conn):
 
 def detect_fts(conn, table):
     """Detect if table has a corresponding FTS virtual table and return it"""
-    sql, params = detect_fts_sql(table)
+    rows = conn.execute(detect_fts_sql(table)).fetchall()
-    rows = conn.execute(sql, params).fetchall()
     if len(rows) == 0:
         return None
     else:
@@ -829,26 +828,18 @@ def detect_fts(conn, table):
 
 
 def detect_fts_sql(table):
-    escaped_table = table.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+    return r"""
-    return (
-        r"""
         select name from sqlite_master
             where rootpage = 0
             and (
-                    sql like :fts_double_quoted escape char(92)
+                sql like '%VIRTUAL TABLE%USING FTS%content="{table}"%'
-                    or sql like :fts_bracket_quoted escape char(92)
+                or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%'
                 or (
-                        tbl_name = :table
+                    tbl_name = "{table}"
                     and sql like '%VIRTUAL TABLE%USING FTS%'
                 )
             )
-        """,
+    """.format(table=table.replace("'", "''"))
-        {
-            "fts_double_quoted": f'%VIRTUAL TABLE%USING FTS%content="{escaped_table}"%',
-            "fts_bracket_quoted": f"%VIRTUAL TABLE%USING FTS%content=[{escaped_table}]%",
-            "table": table,
-        },
-    )
 
 
 def detect_json1(conn=None):
@@ -1566,13 +1557,7 @@ async def row_sql_params_pks(db, table, pk_values):
     if use_rowid:
         select = "rowid, *"
         pks = ["rowid"]
-    wheres = []
+    wheres = [f'"{pk}"=:p{i}' for i, pk in enumerate(pks)]
-    for i, pk in enumerate(pks):
-        escaped_pk = escape_sqlite(pk)
-        # Preserve the historic always-quoted SQL exposed by _extra=query
-        if escaped_pk == pk:
-            escaped_pk = f'"{pk}"'
-        wheres.append(f"{escaped_pk}=:p{i}")
     sql = f"select {select} from {escape_sqlite(table)} where {' AND '.join(wheres)}"
     params = {}
     for i, pk_value in enumerate(pk_values):
@@ -1744,7 +1729,7 @@ def redact_keys(original: dict, key_patterns: Iterable) -> dict:
             return {
                 k: (
                     redact(v)
-                    if not any(pattern in k.casefold() for pattern in key_patterns)
+                    if not any(pattern in k for pattern in key_patterns)
                     else "***"
                 )
                 for k, v in data.items()

datasette/utils/actions_sql.py

@@ -29,15 +29,6 @@ from datasette.utils.permissions import gather_permission_sql_from_hooks
 
 if TYPE_CHECKING:
     from datasette.app import Datasette
-    from datasette.permissions import Action
-
-
-def _child_collation(action: "Action") -> str:
-    """Match resource identity without changing the spelling returned by SQL."""
-    resource_class = action.resource_class
-    if resource_class is not None and resource_class.case_insensitive_child:
-        return "NOCASE"
-    return "BINARY"
 
 
 async def build_allowed_resources_sql(
@@ -158,7 +149,6 @@ async def _build_single_action_sql(
         raise ValueError(f"Unknown action: {action}")
 
     # Get base resources SQL from the resource class
-    child_collation = _child_collation(action_obj)
     base_resources_sql = await action_obj.resource_class.resources_sql(
         datasette, actor=actor
     )
@@ -195,7 +185,7 @@ async def _build_single_action_sql(
         if permission_sql.sql is None:
             continue
         rule_sqls.append(f"""
-            SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
+            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
             """.strip())
@@ -309,9 +299,9 @@ async def _build_single_action_sql(
         query_parts.extend(
             ["anon_child_agg AS ("]
             + _anon_agg(
-                f"parent, child COLLATE {child_collation} AS child,",
+                "parent, child,",
                 "parent IS NOT NULL AND child IS NOT NULL",
-                f"parent, child COLLATE {child_collation}",
+                "parent, child",
             )
             + ["),", "anon_parent_agg AS ("]
             + _anon_agg("parent,", "parent IS NOT NULL AND child IS NULL", "parent")
@@ -392,8 +382,7 @@ async def _build_single_action_sql(
         # Wrap each restriction_sql in a subquery to avoid operator precedence issues
         # with UNION ALL inside the restriction SQL statements
         restriction_intersect = "\nINTERSECT\n".join(
-            f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})"
+            f"SELECT * FROM ({sql})" for sql in restriction_sqls
-            for sql in restriction_sqls
         )
         # Decompose by NULL-pattern so the final filter can use pure-equality
         # EXISTS lookups (satisfiable via automatic indexes) instead of a
@@ -491,7 +480,6 @@ async def build_permission_rules_sql(
     union_parts = []
     all_params = {}
     restriction_sqls = []
-    child_collation = _child_collation(action_obj)
 
     for permission_sql in permission_sqls:
         all_params.update(permission_sql.params or {})
@@ -505,7 +493,7 @@ async def build_permission_rules_sql(
             continue
 
         union_parts.append(f"""
-            SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
+            SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (
                 {permission_sql.sql}
             )
             """.strip())
@@ -576,7 +564,6 @@ async def check_permissions_for_actions(
     verdicts = {}
 
     for i, (action, permission_sqls) in enumerate(zip(unique_actions, gathered)):
-        child_collation = _child_collation(datasette.actions[action])
         prefix = f"a{i}_"
         rule_parts = []
         restriction_parts = []
@@ -602,7 +589,7 @@ async def check_permissions_for_actions(
             if sql is None:
                 continue
             rule_parts.append(
-                f"SELECT parent, child COLLATE {child_collation} AS child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
+                f"SELECT parent, child, allow, reason, '{permission_sql.source}' AS source_plugin FROM (\n{sql}\n)"
             )
 
         if not rule_parts:
@@ -636,8 +623,7 @@ async def check_permissions_for_actions(
         if restriction_parts:
             # Database-level restrictions (parent, NULL) match all children
             restriction_intersect = "\nINTERSECT\n".join(
-                f"SELECT parent, child COLLATE {child_collation} AS child FROM ({sql})"
+                f"SELECT * FROM ({sql})" for sql in restriction_parts
-                for sql in restriction_parts
             )
             ctes.append(f"a{i}_restriction AS (\n{restriction_intersect}\n)")
             verdict_sql = f"""({verdict_sql}) AND EXISTS (
@@ -784,7 +770,6 @@ async def _explain_single_action(
     db = datasette.get_internal_database()
     matched_rules = []
     restrictions = []
-    child_collation = _child_collation(datasette.actions[action])
 
     for permission_sql in permission_sqls:
         params = dict(permission_sql.params or {})
@@ -799,7 +784,7 @@ async def _explain_single_action(
                 SELECT parent, child, allow, reason
                 FROM ({permission_sql.sql}) AS permission_rules
                 WHERE (parent IS NULL OR parent = :{parent_param})
-                  AND (child IS NULL OR child COLLATE {child_collation} = :{child_param})
+                  AND (child IS NULL OR child = :{child_param})
                 """,
                 params,
             )
@@ -826,7 +811,7 @@ async def _explain_single_action(
                     SELECT EXISTS(
                         SELECT 1 FROM ({permission_sql.restriction_sql}) AS restriction_rules
                         WHERE (parent IS NULL OR parent = :{parent_param})
-                          AND (child IS NULL OR child COLLATE {child_collation} = :{child_param})
+                          AND (child IS NULL OR child = :{child_param})
                     ) AS resource_is_in_allowlist
                     """,
                     params,

datasette/utils/asgi.py

@@ -1,4 +1,3 @@
-import asyncio
 import json
 import re
 from http.cookies import Morsel, SimpleCookie
@@ -83,19 +82,6 @@ SAMESITE_VALUES = ("strict", "lax", "none")
 DEFAULT_MAX_POST_BODY_BYTES = 2 * 1024 * 1024  # 2MB
 
 
-class _RequestHeaders(dict):
-    """Incoming headers with lowercase keys and case-insensitive lookups."""
-
-    def __getitem__(self, key):
-        return super().__getitem__(key.lower())
-
-    def get(self, key, default=None):
-        return super().get(key.lower(), default)
-
-    def __contains__(self, key):
-        return super().__contains__(key.lower())
-
-
 class Request:
     def __init__(self, scope, receive, max_post_body_bytes=DEFAULT_MAX_POST_BODY_BYTES):
         self.scope = scope
@@ -125,10 +111,10 @@ class Request:
 
     @property
     def headers(self):
-        return _RequestHeaders(
+        return {
-            (k.decode("latin-1").lower(), v.decode("latin-1"))
+            k.decode("latin-1").lower(): v.decode("latin-1")
             for k, v in self.scope.get("headers") or []
-        )
+        }
 
     @property
     def host(self):
@@ -314,24 +300,12 @@ class AsgiLifespan:
             while True:
                 message = await receive()
                 if message["type"] == "lifespan.startup":
-                    try:
                     for fn in self.on_startup:
                         await fn()
-                    except Exception as e:  # noqa: BLE001
-                        await send(
-                            {"type": "lifespan.startup.failed", "message": str(e)}
-                        )
-                        return
                     await send({"type": "lifespan.startup.complete"})
                 elif message["type"] == "lifespan.shutdown":
-                    try:
                     for fn in self.on_shutdown:
                         await fn()
-                    except Exception as e:  # noqa: BLE001
-                        await send(
-                            {"type": "lifespan.shutdown.failed", "message": str(e)}
-                        )
-                        return
                     await send({"type": "lifespan.shutdown.complete"})
                     return
         else:
@@ -511,8 +485,6 @@ def asgi_static(root_path, chunk_size=4096, headers=None, content_type=None):
             await asgi_send_html(send, "404: File not found", 404)
             return
 
-    # Only the actual static-file handler can bypass dynamic response privacy.
-    inner_static._datasette_static = True
     return inner_static
 
 
@@ -652,23 +624,10 @@ class AsgiRunOnFirstRequest:
         self.asgi = asgi
         self.on_startup = on_startup
         self._started = False
-        # Guards against concurrent early requests interleaving with startup:
-        # without this, several requests could all observe `_started is
-        # False` and proceed before any of them finish running the hooks.
-        self._lock = asyncio.Lock()
 
     async def __call__(self, scope, receive, send):
-        # Leave "lifespan" scope events alone - this shim only exists as a
-        # fallback for hosts that never send them. It wraps AsgiLifespan, so
-        # if it ran on_startup here too, a startup exception would escape
-        # before AsgiLifespan's own try/except got a chance to turn it into
-        # a lifespan.startup.failed message.
-        if scope["type"] != "lifespan" and not self._started:
-            async with self._lock:
-                # Re-check: another request may have finished startup while
-                # we were waiting for the lock.
         if not self._started:
+            self._started = True
             for hook in self.on_startup:
                 await hook()
-                    self._started = True
         return await self.asgi(scope, receive, send)

datasette/utils/multipart.py

@@ -358,13 +358,6 @@ class MultipartParser:
         self.buffer.extend(chunk)
         self._process()
 
-    def close(self) -> None:
-        """Discard completed uploads and any file still being received."""
-        if self.current_file is not None:
-            self.current_file.close()
-            self.current_file = None
-        self.form_data.close()
-
     def _process(self) -> None:
         """Process buffered data."""
         while True:
@@ -584,9 +577,6 @@ class MultipartParser:
     def _finish_part(self) -> None:
         """Finalize current part and add to form data."""
         if self.current_name is None:
-            if self.current_file is not None:
-                self.current_file.close()
-                self.current_file = None
             return
 
         if self.current_filename is not None:
@@ -732,29 +722,12 @@ async def parse_form_data(
         batch_target = 64 * 1024
         batch = bytearray()
 
-        async def run_parser(fn, *args):
-            # Cancellation must not close files while a worker is using them.
-            task = asyncio.create_task(asyncio.to_thread(fn, *args))
-            try:
-                return await asyncio.shield(task)
-            except asyncio.CancelledError as cancelled:
-                try:
-                    while not task.done():
-                        try:
-                            await asyncio.shield(task)
-                        except asyncio.CancelledError:
-                            continue
-                    task.result()
-                finally:
-                    raise cancelled
-
         async def flush_batch() -> None:
             if batch:
                 data = bytes(batch)
                 batch.clear()
-                await run_parser(parser.feed, data)
+                await asyncio.to_thread(parser.feed, data)
 
-        try:
         while True:
             message = await receive()
             message_type = message.get("type")
@@ -771,11 +744,7 @@ async def parse_form_data(
                 break
 
         await flush_batch()
-            return await run_parser(parser.finalize)
+        return await asyncio.to_thread(parser.finalize)
-        except BaseException:
-            # No FormData is returned to the caller to take ownership on failure.
-            await asyncio.to_thread(parser.close)
-            raise
 
     else:
         raise MultipartParseError(

datasette/utils/sql_analysis.py

@@ -1,8 +1,6 @@
-import sys
 from dataclasses import dataclass
 from typing import Literal
 
-from datasette.utils import escape_sqlite
 from datasette.utils.sqlite import SQLiteTableType, sqlite3, sqlite_table_type
 
 SQLOperation = Literal[
@@ -197,16 +195,6 @@ def _allow_authorizer_action(*args):
     return sqlite3.SQLITE_OK
 
 
-def _disable_authorizer(conn):
-    # Python 3.11 added support for unregistering an authorizer using None.
-    # On Python 3.10, None is installed as the callback instead, and the next
-    # statement fails with "not authorized" when sqlite3 tries to call it.
-    if sys.version_info >= (3, 11):
-        conn.set_authorizer(None)
-    else:
-        conn.set_authorizer(_allow_authorizer_action)
-
-
 def analyze_sql_tables(
     conn,
     sql: str,
@@ -220,9 +208,7 @@ def analyze_sql_tables(
 
     This function is synchronous and connection-based. It temporarily installs a
     SQLite authorizer, prepares ``EXPLAIN <sql>``, and returns the operation
-    callbacks observed while SQLite compiles the statement. ``CREATE VIEW`` is
+    callbacks observed while SQLite compiles the statement.
-    additionally executed inside a rolled-back savepoint so its source-table reads
-    can be discovered by analyzing a query against the temporary view.
     """
     operations: dict[OperationKey, set[str]] = {}
 
@@ -495,7 +481,7 @@ def analyze_sql_tables(
                         conn, key.table, schema=key.sqlite_schema
                     )
     finally:
-        _disable_authorizer(conn)
+        conn.set_authorizer(None)
 
     has_schema_operation = any(
         key.target_type in {"table", "index", "view", "trigger", "virtual-table"}
@@ -546,7 +532,7 @@ def analyze_sql_tables(
             return None
         return table_kind_cache[(key.sqlite_schema, key.table)]
 
-    analysis = SQLAnalysis(
+    return SQLAnalysis(
         operations=tuple(
             Operation(
                 operation=key.operation,
@@ -563,58 +549,3 @@ def analyze_sql_tables(
             for key, columns in operations.items()
         )
     )
-
-    # SQLite does not resolve the SELECT body of a view when preparing CREATE
-    # VIEW, so its authorizer does not report reads from the view's source
-    # tables. Temporarily create the view, analyze a query against it (which
-    # does resolve the body), then roll the schema change back. Database-level
-    # callers use an isolated writable connection for this analysis.
-    create_view_operations = tuple(
-        operation
-        for operation in analysis.operations
-        if operation.operation == "create" and operation.target_type == "view"
-    )
-    if not create_view_operations:
-        return analysis
-
-    savepoint = "datasette_analyze_create_view"
-    conn.execute(f"SAVEPOINT {savepoint}")
-    try:
-        conn.execute(sql, params if params is not None else {})
-        dependency_reads = []
-        for view_operation in create_view_operations:
-            if view_operation.sqlite_schema is None or view_operation.table is None:
-                raise sqlite3.OperationalError(
-                    "Could not determine the created view name"
-                )
-            quoted_schema = escape_sqlite(view_operation.sqlite_schema)
-            quoted_view = escape_sqlite(view_operation.table)
-            qualified_view = f"{quoted_schema}.{quoted_view}"
-            view_analysis = analyze_sql_tables(
-                conn,
-                f"SELECT * FROM {qualified_view}",
-                database_name=database_name,
-                schema_to_database=schema_to_database,
-            )
-            dependency_reads.extend(
-                operation
-                for operation in view_analysis.operations
-                if operation.operation == "read"
-                and not (
-                    operation.sqlite_schema == view_operation.sqlite_schema
-                    and operation.table == view_operation.table
-                )
-            )
-    finally:
-        conn.execute(f"ROLLBACK TO {savepoint}")
-        conn.execute(f"RELEASE {savepoint}")
-
-    existing_operations = set(analysis.operations)
-    return SQLAnalysis(
-        operations=analysis.operations
-        + tuple(
-            operation
-            for operation in dependency_reads
-            if operation not in existing_operations
-        )
-    )

datasette/utils/sqlite.py

@@ -15,17 +15,8 @@ if hasattr(sqlite3, "enable_callback_tracebacks"):
 _cached_sqlite_version = None
 _cached_supports_returning = None
 SQLiteTableType = Literal["table", "view", "virtual", "shadow"]
-_SQLITE_IDENTIFIER_RE = (
-    r"""(?:"(?:[^"]|"")*"|'(?:[^']|'')*'|`(?:[^`]|``)*`|\[[^\]]*\]|[^\s.()'"`\[\]]+)"""
-)
 _VIRTUAL_TABLE_MODULE_RE = re.compile(
-    r"^\s*CREATE\s+VIRTUAL\s+TABLE\b\s*(?:IF\s+NOT\s+EXISTS\s+)?"
+    r"\bCREATE\s+VIRTUAL\s+TABLE\b.*?\bUSING\s+([^\s(]+)",
-    + _SQLITE_IDENTIFIER_RE
-    + r"(?:\s*\.\s*"
-    + _SQLITE_IDENTIFIER_RE
-    + r")?\s*\bUSING\b\s*("
-    + _SQLITE_IDENTIFIER_RE
-    + r")",
     re.IGNORECASE | re.DOTALL,
 )
 _VIRTUAL_TABLE_SHADOW_SUFFIXES = {
@@ -92,53 +83,19 @@ def sqlite_table_type(
 ) -> SQLiteTableType | None:
     if supports_table_list():
         try:
-            # Use the "PRAGMA table_list" statement form rather than the
+            query = "select type from pragma_table_list where name = ?"
-            # pragma_table_list(...) table-valued function. The
+            params: tuple[str, ...] = (table,)
-            # table-valued function is resolved like an ordinary relation
-            # name, so an attacker-created table or view literally named
-            # "pragma_table_list" can shadow it and spoof the reported
-            # type (e.g. claiming a virtual table is an ordinary table).
-            # The PRAGMA statement form is a distinct piece of SQL syntax
-            # that always invokes SQLite's built-in pragma, so it cannot
-            # be shadowed by a user-created relation.
             if schema is not None:
-                query = f"PRAGMA {_quote_identifier(schema)}.table_list"
+                query += " and schema = ?"
-            else:
+                params = (table, schema)
-                query = "PRAGMA table_list"
+            row = conn.execute(query, params).fetchone()
-            cursor = conn.execute(query)
+            if row is not None and row[0] in {"table", "view", "virtual", "shadow"}:
-            columns = [description[0] for description in cursor.description]
+                return row[0]
-            for row in cursor.fetchall():
-                record = dict(zip(columns, row))
-                if record.get("name") != table:
-                    continue
-                if schema is not None and record.get("schema") != schema:
-                    continue
-                row_type = record.get("type")
-                if row_type in {"table", "view", "virtual", "shadow"}:
-                    return row_type
         except sqlite3.DatabaseError:
             pass
     return _sqlite_table_type_from_schema(conn, table, schema=schema)
 
 
-def check_structured_write_table(conn, table: str, *, allow_missing=False):
-    """Validate a row-write target on the connection that will perform the write."""
-    # SQLite resolves identifiers case-insensitively. The create API must not
-    # treat a differently cased existing name as a missing table.
-    row = conn.execute(
-        "select name from main.sqlite_master where name = ? collate nocase "
-        "and type in ('table', 'view')",
-        (table,),
-    ).fetchone()
-    if row is None and allow_missing:
-        return
-    if row is not None and sqlite_table_type(conn, row[0]) == "table":
-        return
-    # Virtual table modules can interpret row writes as administrative operations.
-    # Their shadow tables are internal storage, not independently writable data.
-    raise ValueError("Structured writes require an ordinary table")
-
-
 def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]:
     schema_table = _sqlite_schema_table(schema)
     try:
@@ -161,63 +118,6 @@ def sqlite_hidden_table_names(conn, *, schema: str | None = "main") -> list[str]
     return sorted(hidden_tables) + content_fts_tables
 
 
-def sqlite_derived_table_dependencies(
-    conn, *, schema: str | None = "main"
-) -> dict[str, str]:
-    """Return implementation table -> logical/content table dependencies.
-
-    ``PRAGMA table_list`` safely identifies virtual and shadow tables, but
-    does not report which virtual table owns a shadow table or which table is
-    named by an FTS ``content=`` option. Derive those relationships from
-    ``sqlite_master`` DDL and the documented shadow-table suffixes.
-
-    Database errors propagate: failed discovery must not be mistaken for an
-    empty dependency map and cached as permission to skip inheritance.
-    """
-    schema_table = _sqlite_schema_table(schema)
-    rows = conn.execute(
-        f"select name, sql from {schema_table} where type = 'table'"
-    ).fetchall()
-
-    table_names = {row[0] for row in rows}
-    # SQLite identifiers fold ASCII letters only.
-    identifier_case = str.maketrans(
-        "ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz"
-    )
-    canonical_names = {name.translate(identifier_case): name for name in table_names}
-    dependencies = {}
-    for virtual_table, sql in rows:
-        module = _virtual_table_module(sql)
-        if module is None:
-            continue
-
-        # SQLite's documented shadow tables are implementation details of
-        # their logical virtual table.
-        for suffix in _VIRTUAL_TABLE_SHADOW_SUFFIXES.get(module, ()):
-            shadow_table = virtual_table + suffix
-            if shadow_table in table_names:
-                dependencies[shadow_table] = virtual_table
-
-        # An external-content FTS table can expose values fetched from its
-        # content table, so it must also depend on that table's permission.
-        if module in {"fts3", "fts4", "fts5"}:
-            content_table = _fts_external_content_table(sql)
-            if content_table:
-                dependencies[virtual_table] = content_table
-
-        if module in {"fts5vocab", "fts4aux"}:
-            source = _fts_vocabulary_source(sql, module, schema or "main")
-            source = (
-                canonical_names.get(source.translate(identifier_case))
-                if source
-                else None
-            )
-            # An unresolved source is itself derived, so the one-hop policy denies it.
-            dependencies[virtual_table] = source or virtual_table
-
-    return dependencies
-
-
 def _sqlite_table_type_from_schema(
     conn,
     table: str,
@@ -284,151 +184,10 @@ def _quote_identifier(value: str) -> str:
 def _virtual_table_module(sql: str | None) -> str | None:
     if not sql:
         return None
-    match = _VIRTUAL_TABLE_MODULE_RE.search(_strip_sql_comments(sql))
-    if match is None:
-        return None
-    return _unquote_sql_value(match.group(1)).lower()
-
-
-def _fts_external_content_table(sql: str | None) -> str | None:
-    """Extract the external ``content=`` table from an FTS declaration."""
-    if not sql:
-        return None
-    sql = _strip_sql_comments(sql)
     match = _VIRTUAL_TABLE_MODULE_RE.search(sql)
     if match is None:
         return None
-    open_paren = sql.find("(", match.end())
+    return match.group(1).strip("\"'[]`").lower()
-    if open_paren == -1:
-        return None
-    close_paren = sql.rfind(")")
-    if close_paren <= open_paren:
-        return None
-
-    for argument in _split_sql_arguments(sql[open_paren + 1 : close_paren]):
-        key, separator, value = argument.partition("=")
-        if not separator or key.strip().lower() != "content":
-            continue
-        return _unquote_sql_value(value.strip())
-    return None
-
-
-def _fts_vocabulary_source(sql: str, module: str, schema: str) -> str | None:
-    """Resolve a vocabulary source within the current SQLite schema.
-
-    Cross-schema sources cannot be represented by the dependency map and
-    are conservatively left unresolved.
-    """
-    sql = _strip_sql_comments(sql)
-    match = _VIRTUAL_TABLE_MODULE_RE.search(sql)
-    if match is None:
-        return None
-    start = sql.find("(", match.end())
-    end = sql.rfind(")")
-    if start < 0 or end <= start:
-        return None
-    arguments = [
-        _unquote_sql_value(arg.strip())
-        for arg in _split_sql_arguments(sql[start + 1 : end])
-    ]
-    expected = 2 if module == "fts5vocab" else 1
-    if len(arguments) == expected:
-        return arguments[0]
-    if len(arguments) == expected + 1 and arguments[0].lower() == schema.lower():
-        return arguments[1]
-    return None
-
-
-def _split_sql_arguments(arguments: str) -> list[str]:
-    """Split comma-separated SQLite arguments without splitting quoted text."""
-    parts = []
-    start = 0
-    quote = None
-    closing_quote = None
-    index = 0
-    while index < len(arguments):
-        char = arguments[index]
-        if quote is None:
-            if char in {"'", '"', "`", "["}:
-                quote = char
-                closing_quote = "]" if char == "[" else char
-            elif char == ",":
-                parts.append(arguments[start:index])
-                start = index + 1
-        elif char == closing_quote:
-            # Single/double/backtick quoting escapes the delimiter by
-            # doubling it. Square-bracket identifiers do not.
-            if (
-                quote != "["
-                and index + 1 < len(arguments)
-                and arguments[index + 1] == closing_quote
-            ):
-                index += 1
-            else:
-                quote = None
-                closing_quote = None
-        index += 1
-    parts.append(arguments[start:])
-    return parts
-
-
-def _strip_sql_comments(sql: str) -> str:
-    """Remove SQLite comments while preserving quoted strings/identifiers."""
-    output = []
-    quote = None
-    closing_quote = None
-    index = 0
-    while index < len(sql):
-        char = sql[index]
-        next_char = sql[index + 1] if index + 1 < len(sql) else ""
-        if quote is None:
-            if char in {"'", '"', "`", "["}:
-                quote = char
-                closing_quote = "]" if char == "[" else char
-                output.append(char)
-            elif char == "-" and next_char == "-":
-                index += 2
-                while index < len(sql) and sql[index] not in "\r\n":
-                    index += 1
-                output.append(" ")
-                continue
-            elif char == "/" and next_char == "*":
-                index += 2
-                while index + 1 < len(sql) and sql[index : index + 2] != "*/":
-                    index += 1
-                index = min(index + 2, len(sql))
-                output.append(" ")
-                continue
-            else:
-                output.append(char)
-        else:
-            output.append(char)
-            if char == closing_quote:
-                if (
-                    quote != "["
-                    and index + 1 < len(sql)
-                    and sql[index + 1] == closing_quote
-                ):
-                    output.append(sql[index + 1])
-                    index += 1
-                else:
-                    quote = None
-                    closing_quote = None
-        index += 1
-    return "".join(output)
-
-
-def _unquote_sql_value(value: str) -> str:
-    if len(value) < 2:
-        return value
-    pairs = {"'": "'", '"': '"', "`": "`", "[": "]"}
-    closing = pairs.get(value[0])
-    if closing is None or value[-1] != closing:
-        return value
-    unquoted = value[1:-1]
-    if value[0] != "[":
-        unquoted = unquoted.replace(closing * 2, closing)
-    return unquoted
 
 
 def _is_fts_content_virtual_table(sql: str | None) -> bool:

datasette/utils/testing.py

@@ -4,7 +4,7 @@ from urllib.parse import urlencode
 from asgiref.sync import async_to_sync
 
 # These wrapper classes pre-date the introduction of
-# datasette.client and httpx2 to Datasette. They could
+# datasette.client and httpx to Datasette. They could
 # be removed if the Datasette tests are modified to
 # call datasette.client directly.
 

datasette/version.py

@@ -1,2 +1,2 @@
-__version__ = "1.0a41"
+__version__ = "1.0a38"
 __version_info__ = tuple(__version__.split("."))

datasette/views/base.py

@@ -35,7 +35,7 @@ class DatasetteError(Exception):
         self.error_dict = error_dict or {}
         self.status = status
         self.message_is_html = message_is_html
-        # Plain text used for JSON and CSV error responses when message is HTML
+        # Plain text used for JSON error responses when message is HTML
         self.plain_message = plain_message
 
 

datasette/views/database.py

@@ -40,11 +40,7 @@ from datasette.write_sql import QueryWriteRejected
 
 from . import Context
 from .base import DatasetteError, View, stream_csv
-from .query_helpers import (
+from .query_helpers import _ensure_stored_query_execution_permissions, _table_columns
-    _block_framing,
-    _ensure_stored_query_execution_permissions,
-    _table_columns,
-)
 from .table_create_alter import _create_table_ui_context
 from .table_extras import (
     QueryExtraContext,
@@ -861,8 +857,7 @@ class QueryView(View):
                 raise DatasetteError("?sql= is required", status=400)
 
             async def fetch_data_for_csv(request, _next=None):
-                # Reuse the trusted magic parameter values prepared above.
+                results = await db.execute(sql, params, truncate=True)
-                results = await db.execute(sql, params_for_query, truncate=True)
                 data = {"rows": results.rows, "columns": results.columns}
                 return data, None, None
 
@@ -1145,8 +1140,6 @@ class QueryView(View):
             assert False, f"Invalid format: {format_}"
         if datasette.cors:
             add_cors_headers(r.headers)
-        if stored_query_write and format_ == "html":
-            _block_framing(r)
         return r
 
 

datasette/views/execute_write.py

@@ -1,7 +1,6 @@
 import re
 from urllib.parse import urlencode
 
-from datasette.database import QueryInterrupted
 from datasette.resources import DatabaseResource
 from datasette.utils import UNSTABLE_API_MESSAGE, sqlite3
 from datasette.utils.asgi import Response
@@ -385,7 +384,7 @@ class ExecuteWriteView(BaseView):
         try:
             execute_write_kwargs = {"request": request}
             cursor = await db.execute_write(sql, params, **execute_write_kwargs)
-        except (QueryInterrupted, sqlite3.DatabaseError) as ex:
+        except sqlite3.DatabaseError as ex:
             message = str(ex)
             if wants_json:
                 return _block_framing(Response.error([message], 400))

datasette/views/row.py

@@ -28,11 +28,9 @@ from datasette.utils import (
     path_with_format,
     path_with_removed_args,
     sqlite3,
-    tilde_decode,
     to_css_class,
 )
 from datasette.utils.asgi import Forbidden, NotFound, PayloadTooLarge, Response
-from datasette.utils.sqlite import check_structured_write_table
 
 from . import Context, from_extra
 from .base import BaseView, DatasetteError, stream_csv
@@ -139,12 +137,6 @@ class RowContext(Context):
     )
 
 
-async def _database_and_table_resource_from_request(datasette, request):
-    db = await datasette.resolve_database(request)
-    table = tilde_decode(request.url_vars["table"])
-    return db, table, TableResource(database=db.name, table=table)
-
-
 class RowView(BaseView):
     name = "row"
 
@@ -271,7 +263,7 @@ class RowView(BaseView):
         if ttl is None or not ttl.isdigit():
             ttl = self.ds.setting("default_cache_ttl")
 
-        return self.set_response_headers(response, ttl, request)
+        return self.set_response_headers(response, ttl)
 
     async def html(self, request, data, extra_template_data, templates):
         extras = {}
@@ -384,17 +376,9 @@ class RowView(BaseView):
             },
         )
 
-    def set_response_headers(self, response, ttl, request=None):
+    def set_response_headers(self, response, ttl):
-        private = getattr(request, "_datasette_private_response", False)
         # Set far-future cache expiry
         if self.ds.cache_headers and response.status == 200:
-            if private:
-                # This response is only visible to the current actor (denied
-                # to anonymous requests), so it must never be stored by a
-                # shared cache/CDN - and ?_ttl= must not override that.
-                response.headers["Cache-Control"] = "private, no-store"
-                response.headers["Vary"] = "Cookie"
-            else:
             ttl = int(ttl)
             if ttl == 0:
                 ttl_header = "no-cache"
@@ -407,27 +391,21 @@ class RowView(BaseView):
         return response
 
     async def data(self, request, default_labels=False):
-        db, table, resource = await _database_and_table_resource_from_request(
+        resolved = await self.ds.resolve_row(request)
-            self.ds, request
+        db = resolved.db
-        )
         database = db.name
+        table = resolved.table
+        pk_values = resolved.pk_values
 
-        # Check the URL resource before resolving the row, so a denied request
+        # Ensure user has permission to view this row
-        # cannot distinguish an existing primary key from a missing one.
         visible, private = await self.ds.check_visibility(
             request.actor,
             action="view-table",
-            resource=resource,
+            resource=TableResource(database=database, table=table),
         )
         if not visible:
             raise Forbidden("You do not have permission to view this table")
-        # Record whether this response is private (visible to this actor
-        # only) so set_response_headers() can set appropriate Cache-Control
-        # headers, regardless of which output format ends up being rendered.
-        request._datasette_private_response = private
 
-        resolved = await self.ds.resolve_row(request)
-        pk_values = resolved.pk_values
         results = await resolved.db.execute(
             resolved.sql, resolved.params, truncate=True
         )
@@ -504,8 +482,8 @@ class RowView(BaseView):
             for row in display_rows:
                 for cell in row:
                     if cell["column"] in pk_set:
-                        cell["value"] = markupsafe.Markup("<strong>{}</strong>").format(
+                        cell["value"] = markupsafe.Markup(
-                            cell["value"]
+                            "<strong>{}</strong>".format(cell["value"])
                         )
 
             label_column = await db.label_column_for_table(table) if is_table else None
@@ -578,7 +556,7 @@ class RowView(BaseView):
                 "private": private,
                 "columns": reordered_columns,
                 "foreign_key_tables": await self.foreign_key_tables(
-                    database, table, pk_values, actor=request.actor
+                    database, table, pk_values
                 ),
                 "database_color": db.color,
                 "display_columns": display_columns,
@@ -655,23 +633,12 @@ class RowView(BaseView):
             ),
         )
 
-    async def foreign_key_tables(self, database, table, pk_values, *, actor):
+    async def foreign_key_tables(self, database, table, pk_values):
         if len(pk_values) != 1:
             return []
         db = self.ds.databases[database]
         all_foreign_keys = await db.get_all_foreign_keys()
-        foreign_keys = []
+        foreign_keys = all_foreign_keys[table]["incoming"]
-        table_permissions = {}
-        for fk in all_foreign_keys[table]["incoming"]:
-            other_table = fk["other_table"]
-            if other_table not in table_permissions:
-                table_permissions[other_table] = await self.ds.allowed(
-                    action="view-table",
-                    resource=TableResource(database=database, table=other_table),
-                    actor=actor,
-                )
-            if table_permissions[other_table]:
-                foreign_keys.append(fk)
         if len(foreign_keys) == 0:
             return []
 
@@ -728,24 +695,9 @@ def _truncated_row_flash_label(label):
     return label[: ROW_FLASH_LABEL_MAX_LENGTH - 1] + "\u2026"
 
 
-async def _row_flash_message(
+async def _row_flash_message(db, action, resolved, row=None):
-    datasette, request, action, resolved, row=None, *, refresh_row=False
-):
     pk_label = ", ".join(resolved.pk_values)
-    # Mutation permission does not grant access to stored row labels.
+    label_column = await db.label_column_for_table(resolved.table)
-    if not await datasette.allowed(
-        action="view-table",
-        resource=TableResource(database=resolved.db.name, table=resolved.table),
-        actor=request.actor,
-    ):
-        return f"{action} row {pk_label}"
-
-    if refresh_row and row is None:
-        results = await resolved.db.execute(
-            resolved.sql, resolved.params, truncate=True
-        )
-        row = results.first()
-    label_column = await resolved.db.label_column_for_table(resolved.table)
     label = row_label_from_label_column(row or resolved.row, label_column)
     if label:
         label = _truncated_row_flash_label(label)
@@ -758,28 +710,22 @@ async def _resolve_row_and_check_permission(datasette, request, permission):
     from datasette.app import DatabaseNotFound, RowNotFound, TableNotFound
 
     try:
-        _, _, resource = await _database_and_table_resource_from_request(
+        resolved = await datasette.resolve_row(request)
-            datasette, request
-        )
     except DatabaseNotFound as e:
         return False, Response.error([f"Database not found: {e.database_name}"], 404)
-
-    # Check the URL resource before resolving the row, so a denied request
-    # cannot distinguish an existing primary key from a missing one.
-    if not await datasette.allowed(
-        action=permission,
-        resource=resource,
-        actor=request.actor,
-    ):
-        return False, Response.error(["Permission denied"], 403)
-
-    try:
-        resolved = await datasette.resolve_row(request)
     except TableNotFound as e:
         return False, Response.error([f"Table not found: {e.table}"], 404)
     except RowNotFound as e:
         return False, Response.error([f"Record not found: {e.pk_values}"], 404)
 
+    # Ensure user has permission to delete this row
+    if not await datasette.allowed(
+        action=permission,
+        resource=TableResource(database=resolved.db.name, table=resolved.table),
+        actor=request.actor,
+    ):
+        return False, Response.error(["Permission denied"], 403)
+
     return True, resolved
 
 
@@ -798,7 +744,6 @@ class RowDeleteView(BaseView):
 
         # Delete table
         def delete_row(conn):
-            check_structured_write_table(conn, resolved.table)
             sqlite_utils.Database(conn)[resolved.table].delete(resolved.pk_values)
 
         try:
@@ -820,7 +765,7 @@ class RowDeleteView(BaseView):
             table_url = self.ds.urls.table(resolved.db.name, resolved.table)
             self.ds.add_message(
                 request,
-                await _row_flash_message(self.ds, request, "Deleted", resolved),
+                await _row_flash_message(resolved.db, "Deleted", resolved),
                 self.ds.INFO,
             )
             return Response.json({"ok": True, "redirect": str(table_url)}, status=200)
@@ -881,7 +826,6 @@ class RowUpdateView(BaseView):
             return Response.error(["Permission denied for alter-table"], 403)
 
         def update_row(conn):
-            check_structured_write_table(conn, resolved.table)
             sqlite_utils.Database(conn)[resolved.table].update(
                 resolved.pk_values, update, alter=alter
             )
@@ -894,14 +838,7 @@ class RowUpdateView(BaseView):
 
         result = {"ok": True}
         returned_row = None
-        # Only read back and disclose the stored row if the actor is also
+        if data.get("return"):
-        # allowed to view this table - update-row alone must not be usable
-        # to read data the actor cannot otherwise see.
-        if data.get("return") and await self.ds.allowed(
-            action="view-table",
-            resource=TableResource(database=resolved.db.name, table=resolved.table),
-            actor=request.actor,
-        ):
             results = await resolved.db.execute(
                 resolved.sql, resolved.params, truncate=True
             )
@@ -918,15 +855,16 @@ class RowUpdateView(BaseView):
         )
 
         if request.args.get("_message"):
+            message_row = returned_row
+            if message_row is None:
+                results = await resolved.db.execute(
+                    resolved.sql, resolved.params, truncate=True
+                )
+                message_row = results.first()
             self.ds.add_message(
                 request,
                 await _row_flash_message(
-                    self.ds,
+                    resolved.db, "Updated", resolved, row=message_row
-                    request,
-                    "Updated",
-                    resolved,
-                    row=returned_row,
-                    refresh_row=True,
                 ),
                 self.ds.INFO,
             )

datasette/views/special.py

@@ -311,7 +311,6 @@ class AllowedResourcesView(BaseView):
     has_json_alternate = False
 
     async def get(self, request):
-        await self.ds.ensure_permission(action="view-instance", actor=request.actor)
         await self.ds.refresh_schemas()
 
         # Check if user has permissions-debug (to show sensitive fields)
@@ -797,8 +796,6 @@ class CreateTokenView(BaseView):
             raise Forbidden(
                 "Token authentication cannot be used to create additional tokens"
             )
-        if "_r" in request.actor:
-            raise Forbidden("Restricted actors cannot create API tokens")
 
     async def shared(self, request):
         self.check_permission(request)
@@ -876,11 +873,6 @@ class CreateTokenView(BaseView):
                 else:
                     errors.append("Invalid expire duration unit")
 
-        if errors:
-            context = await self.shared(request)
-            context["errors"] = errors
-            return await self.render(["create_token.html"], request, context)
-
         # Are there any restrictions?
         from datasette.tokens import TokenRestrictions
 
@@ -1269,21 +1261,14 @@ class SchemaBaseView(BaseView):
 
     has_json_alternate = False
 
-    async def get_database_schema(self, database_name, actor):
+    async def get_database_schema(self, database_name):
         """Get schema SQL for a database."""
         db = self.ds.databases[database_name]
-        allowed_tables_page = await self.ds.allowed_resources(
-            "view-table", actor, parent=database_name
-        )
-        allowed_table_names = {
-            resource.child async for resource in allowed_tables_page.all()
-        }
         result = await db.execute(
-            "select tbl_name, sql from sqlite_master where sql is not null"
+            "select group_concat(sql, ';' || CHAR(10)) as schema from sqlite_master where sql is not null"
-        )
-        return ";\n".join(
-            row["sql"] for row in result.rows if row["tbl_name"] in allowed_table_names
         )
+        row = result.first()
+        return row["schema"] if row and row["schema"] else ""
 
     def format_json_response(self, data):
         """Format data as JSON response with CORS headers if needed."""
@@ -1345,7 +1330,7 @@ class InstanceSchemaView(SchemaBaseView):
         # Get schema for each database
         schemas = []
         for database_name in allowed_databases:
-            schema = await self.get_database_schema(database_name, request.actor)
+            schema = await self.get_database_schema(database_name)
             schemas.append({"database": database_name, "schema": schema})
 
         if format_ == "json":
@@ -1386,7 +1371,7 @@ class DatabaseSchemaView(SchemaBaseView):
         if database_name not in self.ds.databases:
             return self.format_error_response("Database not found", format_)
 
-        schema = await self.get_database_schema(database_name, request.actor)
+        schema = await self.get_database_schema(database_name)
 
         if format_ == "json":
             return self.format_json_response(
@@ -1425,8 +1410,7 @@ class TableSchemaView(SchemaBaseView):
         # Get schema for the table
         db = self.ds.databases[database_name]
         result = await db.execute(
-            "select sql from sqlite_master where name = ? "
+            "select sql from sqlite_master where name = ? and sql is not null",
-            "and type in ('table', 'view') and sql is not null",
             [table_name],
         )
         row = result.first()

datasette/views/stored_queries.py

@@ -279,7 +279,7 @@ class QueryCreateView(BaseView):
             ),
         )
         response.status = status
-        return _block_framing(response)
+        return response
 
     async def get(self, request):
         db = await self.ds.resolve_database(request)
@@ -527,7 +527,7 @@ class QueryEditView(BaseView):
             ),
         )
         response.status = status
-        return _block_framing(response)
+        return response
 
     async def get(self, request):
         db, query_name, existing = await self._load(request)
@@ -639,8 +639,7 @@ class QueryDeleteView(BaseView):
             return Response.error(
                 ["Trusted queries cannot be deleted using the API"], 403
             )
-        return _block_framing(
+        return await self.render(
-            await self.render(
             ["query_delete.html"],
             request,
             {
@@ -650,7 +649,6 @@ class QueryDeleteView(BaseView):
                 "query_url": self.ds.urls.table(db.name, query_name),
             },
         )
-        )
 
     async def post(self, request):
         db, query_name, existing = await self._load(request)

datasette/views/table.py

@@ -1,7 +1,6 @@
 import asyncio
 import itertools
 import json
-import time
 import urllib
 import urllib.parse
 from dataclasses import dataclass, field
@@ -58,7 +57,6 @@ from datasette.utils.asgi import (
     Request,
     Response,
 )
-from datasette.utils.sqlite import check_structured_write_table
 
 from . import Context, from_extra
 from .base import BaseView, DatasetteError, stream_csv
@@ -672,7 +670,7 @@ async def display_columns_and_rows(
     }
     pks = await db.primary_keys(table_name)
     pks_for_display = pks
-    if not pks_for_display and not await db.view_exists(table_name):
+    if not pks_for_display:
         pks_for_display = ["rowid"]
     label_column = None
     if link_column:
@@ -903,7 +901,7 @@ async def display_columns_and_rows(
             columns = [col for col in columns if col["name"] != pks[0]]
             first_column = {
                 "name": pks[0],
-                "sortable": pks[0] in sortable_columns,
+                "sortable": len(pks) == 1,
                 "is_pk": True,
                 "type": column_details[pks[0]].type,
                 "notnull": column_details[pks[0]].notnull,
@@ -1128,7 +1126,6 @@ class TableInsertView(BaseView):
             row_pk_values_for_later = [tuple(row[pk] for pk in pks) for row in rows]
 
         def insert_or_upsert_rows(conn):
-            check_structured_write_table(conn, table_name)
             table = sqlite_utils.Database(conn)[table_name]
             kwargs = {}
             if upsert:
@@ -1160,32 +1157,17 @@ class TableInsertView(BaseView):
             # TODO: narrow to expected write errors so Datasette bugs surface as 500s
             return Response.error([str(e)])
         result = {"ok": True}
-        # Only read back and disclose stored rows if the actor is also
-        # allowed to view this table - insert-row/update-row alone must
-        # not be usable to read data the actor cannot otherwise see.
-        if should_return and not await self.ds.allowed(
-            action="view-table",
-            resource=TableResource(database=database_name, table=table_name),
-            actor=request.actor,
-        ):
-            should_return = False
         if should_return:
             if upsert:
                 # Fetch based on initial input IDs
                 where_clause = " OR ".join(
-                    [
+                    ["({})".format(" AND ".join(f"{pk} = ?" for pk in pks))]
-                        "({})".format(
-                            " AND ".join(f"{escape_sqlite(pk)} = ?" for pk in pks)
-                        )
-                    ]
                     * len(row_pk_values_for_later)
                 )
                 args = list(itertools.chain.from_iterable(row_pk_values_for_later))
                 fetched_rows = await db.execute(
-                    "select {}* from {} where {}".format(
+                    "select {}* from [{}] where {}".format(
-                        "rowid, " if pks == ["rowid"] else "",
+                        "rowid, " if pks == ["rowid"] else "", table_name, where_clause
-                        escape_sqlite(table_name),
-                        where_clause,
                     ),
                     args,
                 )
@@ -1400,9 +1382,7 @@ class TableDropView(BaseView):
                     "database": database_name,
                     "table": table_name,
                     "row_count": (
-                        await db.execute(
+                        await db.execute(f"select count(*) from [{table_name}]")
-                            f"select count(*) from {escape_sqlite(table_name)}"
-                        )
                     ).single_value(),
                     "message": 'Pass "confirm": true to confirm',
                 },
@@ -1411,9 +1391,7 @@ class TableDropView(BaseView):
 
         # Drop table
         def drop_table(conn):
-            table = sqlite_utils.Database(conn)[table_name]
+            sqlite_utils.Database(conn)[table_name].drop()
-            table.disable_fts()
-            table.drop()
 
         await db.execute_write_fn(drop_table, request=request)
         await self.ds.track_event(
@@ -1429,42 +1407,6 @@ class TableDropView(BaseView):
         return Response.json({"ok": True}, status=200)
 
 
-class TableCountView(BaseView):
-    name = "table-count"
-
-    async def post(self, request):
-        try:
-            return await self.count(request)
-        except (NotFound, Forbidden, BadRequest, DatasetteError) as ex:
-            return Response.error(str(ex), status=ex.status)
-
-    async def count(self, request):
-        resolved = await self.ds.resolve_table(request)
-        visible, _private = await self.ds.check_visibility(
-            request.actor,
-            action="view-table",
-            resource=TableResource(database=resolved.db.name, table=resolved.table),
-        )
-        if not visible:
-            raise Forbidden("You do not have permission to view this table")
-        _, where_clauses, params, _, _ = await _table_filters(
-            self.ds, request, resolved.db.name, resolved.table
-        )
-        sql = f"select count(*) from {escape_sqlite(resolved.table)}"
-        if where_clauses:
-            sql += " where " + " and ".join(where_clauses)
-        try:
-            results = await resolved.db.execute(sql, params)
-        except QueryInterrupted:
-            return Response.error("Count query timed out", status=400)
-        except (sqlite3.OperationalError, InvalidSql) as ex:
-            return Response.error(str(ex), status=400)
-        return Response.json(
-            {"ok": True, "count": results.single_value()},
-            headers={"Cache-Control": "no-store"},
-        )
-
-
 class TableFragmentView(BaseView):
     name = "table-fragment"
 
@@ -1751,16 +1693,7 @@ async def table_view(datasette, request):
     if ttl is None or not ttl.isdigit():
         ttl = datasette.setting("default_cache_ttl")
 
-    private = getattr(request, "_datasette_private_response", False)
-
     if datasette.cache_headers and response.status == 200:
-        if private:
-            # This response is only visible to the current actor (denied to
-            # anonymous requests), so it must never be stored by a shared
-            # cache/CDN - and ?_ttl= must not be able to override that.
-            response.headers["Cache-Control"] = "private, no-store"
-            response.headers["Vary"] = "Cookie"
-        else:
         ttl = int(ttl)
         if ttl == 0:
             ttl_header = "no-cache"
@@ -1802,7 +1735,6 @@ async def table_view_traced(datasette, request):
         context_for_html_hack = True
         default_labels = True
 
-    start = time.perf_counter()
     view_data = await table_view_data(
         datasette,
         request,
@@ -1813,7 +1745,6 @@ async def table_view_traced(datasette, request):
     )
     if isinstance(view_data, Response):
         return view_data
-    query_ms = (time.perf_counter() - start) * 1000
     data, rows, columns, _expanded_columns, sql, next_url = view_data
 
     # Handle formats from plugins
@@ -1960,7 +1891,7 @@ async def table_view_traced(datasette, request):
                 resource=DatabaseResource(database=resolved.db.name),
                 actor=request.actor,
             ),
-            query_ms=query_ms,
+            query_ms=1.2,
             select_templates=[
                 f"{'*' if template_name == template.name else ''}{template_name}"
                 for template_name in templates
@@ -1992,47 +1923,6 @@ async def table_view_traced(datasette, request):
     return r
 
 
-async def _table_filters(datasette, request, database_name, table_name):
-    # Arguments that start with _ and don't contain a __ are
-    # special - things like ?_search= - and should not be
-    # treated as filters.
-    filter_args = []
-    for key in request.args:
-        if not (key.startswith("_") and "__" not in key):
-            for v in request.args.getlist(key):
-                filter_args.append((key, v))
-
-    # Build where clauses from query string arguments
-    filters = Filters(sorted(filter_args))
-    where_clauses, params = filters.build_where_clauses(table_name)
-
-    # Execute filters_from_request plugin hooks - including the default
-    # ones that live in datasette/filters.py
-    extra_context_from_filters = {}
-    extra_human_descriptions = []
-
-    for hook in pm.hook.filters_from_request(
-        request=request,
-        table=table_name,
-        database=database_name,
-        datasette=datasette,
-    ):
-        filter_arguments = await await_me_maybe(hook)
-        if filter_arguments:
-            where_clauses.extend(filter_arguments.where_clauses)
-            params.update(filter_arguments.params)
-            extra_human_descriptions.extend(filter_arguments.human_descriptions)
-            extra_context_from_filters.update(filter_arguments.extra_context)
-
-    return (
-        filters,
-        where_clauses,
-        params,
-        extra_human_descriptions,
-        extra_context_from_filters,
-    )
-
-
 async def table_view_data(
     datasette,
     request,
@@ -2057,10 +1947,6 @@ async def table_view_data(
     )
     if not visible:
         raise Forbidden("You do not have permission to view this table")
-    # Record whether this response is private (visible to this actor only)
-    # so the outer table_view() can set appropriate Cache-Control headers,
-    # regardless of which output format ends up being rendered.
-    request._datasette_private_response = private
 
     # Redirect based on request.args, if necessary
     redirect_response = await _redirect_if_needed(datasette, request, resolved)
@@ -2111,13 +1997,36 @@ async def table_view_data(
 
     table_metadata = await datasette.table_config(database_name, table_name)
 
-    (
+    # Arguments that start with _ and don't contain a __ are
-        filters,
+    # special - things like ?_search= - and should not be
-        where_clauses,
+    # treated as filters.
-        params,
+    filter_args = []
-        extra_human_descriptions,
+    for key in request.args:
-        extra_context_from_filters,
+        if not (key.startswith("_") and "__" not in key):
-    ) = await _table_filters(datasette, request, database_name, table_name)
+            for v in request.args.getlist(key):
+                filter_args.append((key, v))
+
+    # Build where clauses from query string arguments
+    filters = Filters(sorted(filter_args))
+    where_clauses, params = filters.build_where_clauses(table_name)
+
+    # Execute filters_from_request plugin hooks - including the default
+    # ones that live in datasette/filters.py
+    extra_context_from_filters = {}
+    extra_human_descriptions = []
+
+    for hook in pm.hook.filters_from_request(
+        request=request,
+        table=table_name,
+        database=database_name,
+        datasette=datasette,
+    ):
+        filter_arguments = await await_me_maybe(hook)
+        if filter_arguments:
+            where_clauses.extend(filter_arguments.where_clauses)
+            params.update(filter_arguments.params)
+            extra_human_descriptions.extend(filter_arguments.human_descriptions)
+            extra_context_from_filters.update(filter_arguments.extra_context)
 
     # Deal with custom sort orders
     sortable_columns = await _sortable_columns_for_table(
@@ -2315,6 +2224,8 @@ async def table_view_data(
                 new_rows.append(new_row)
             rows = new_rows
 
+    _next = request.args.get("_next")
+
     # Pagination next link
     next_value, next_url = await _next_value_and_url(
         datasette,
@@ -2519,12 +2430,9 @@ async def _next_value_and_url(
             except IndexError:
                 # sort/sort_desc column missing from SELECT - look up value by PK instead
                 prefix_where_clause = " and ".join(
-                    f"{escape_sqlite(pk)} = :pk{i}" for i, pk in enumerate(pks)
+                    f"[{pk}] = :pk{i}" for i, pk in enumerate(pks)
-                )
-                prefix_lookup_sql = (
-                    f"select {escape_sqlite(sort or sort_desc)} "
-                    f"from {escape_sqlite(table_name)} where {prefix_where_clause}"
                 )
+                prefix_lookup_sql = f"select [{sort or sort_desc}] from [{table_name}] where {prefix_where_clause}"
                 prefix = (
                     await db.execute(
                         prefix_lookup_sql,

datasette/views/table_create_alter.py

@@ -27,15 +27,7 @@ from datasette.utils import (
     table_column_details,
 )
 from datasette.utils.asgi import NotFound, PayloadTooLarge, Response
-from datasette.utils.permissions import (
+from datasette.utils.sqlite import sqlite_hidden_table_names
-    SKIP_PERMISSION_CHECKS,
-    gather_permission_sql_from_hooks,
-    resolve_permissions_with_candidates,
-)
-from datasette.utils.sqlite import (
-    check_structured_write_table,
-    sqlite_hidden_table_names,
-)
 
 from .base import BaseView
 
@@ -130,30 +122,6 @@ def _public_foreign_key_target(target):
     }
 
 
-async def _filter_visible_foreign_key_targets(datasette, actor, database_name, targets):
-    if not targets:
-        return []
-
-    permission_sqls = await gather_permission_sql_from_hooks(
-        datasette=datasette,
-        actor=actor,
-        action="view-table",
-    )
-    if permission_sqls is SKIP_PERMISSION_CHECKS:
-        return targets
-
-    candidate_tables = list(dict.fromkeys(target["fk_table"] for target in targets))
-    permission_rows = await resolve_permissions_with_candidates(
-        datasette.get_internal_database(),
-        actor,
-        permission_sqls,
-        [(database_name, table_name) for table_name in candidate_tables],
-        "view-table",
-    )
-    visible_tables = {row["child"] for row in permission_rows if bool(row["allow"])}
-    return [target for target in targets if target["fk_table"] in visible_tables]
-
-
 def _singular(name):
     if name.endswith("ies") and len(name) > 3:
         return name[:-3] + "y"
@@ -853,18 +821,16 @@ class TableCreateView(BaseView):
         ignore = create_request.ignore
         replace = create_request.replace
 
-        table_name = create_request.table
-        table_exists = await db.table_exists(table_name)
-        table_resource = TableResource(database=database_name, table=table_name)
-
         # Replacing rows requires update-row permission
         if replace and not await self.ds.allowed(
             action="update-row",
-            resource=table_resource,
+            resource=DatabaseResource(database=database_name),
             actor=request.actor,
         ):
             return Response.error(["Permission denied: need update-row"], 403)
 
+        table_name = create_request.table
+        table_exists = await db.table_exists(table_name)
         columns = create_request.columns
         rows = create_request.rows_list
 
@@ -872,7 +838,7 @@ class TableCreateView(BaseView):
             # Must have insert-row permission
             if not await self.ds.allowed(
                 action="insert-row",
-                resource=table_resource,
+                resource=DatabaseResource(database=database_name),
                 actor=request.actor,
             ):
                 return Response.error(["Permission denied: need insert-row"], 403)
@@ -891,7 +857,7 @@ class TableCreateView(BaseView):
                 if create_request.alter:
                     if not await self.ds.allowed(
                         action="alter-table",
-                        resource=table_resource,
+                        resource=DatabaseResource(database=database_name),
                         actor=request.actor,
                     ):
                         return Response.error(
@@ -927,7 +893,6 @@ class TableCreateView(BaseView):
             )
 
         def create_table(conn):
-            check_structured_write_table(conn, table_name, allow_missing=True)
             db_for_write = sqlite_utils.Database(conn)
             table = db_for_write[table_name]
             if rows:
@@ -1047,9 +1012,6 @@ class DatabaseForeignKeyTargetsView(BaseView):
             for target in (await db.execute(FOREIGN_KEY_TARGETS_SQL)).dicts()
             if target["fk_table"] not in hidden_tables
         ]
-        targets = await _filter_visible_foreign_key_targets(
-            self.ds, request.actor, database_name, targets
-        )
         return Response.json(
             {
                 "ok": True,
@@ -1088,15 +1050,6 @@ class TableForeignKeySuggestionsView(BaseView):
         source_columns, targets, current_by_column = await db.execute_fn(
             lambda conn: _foreign_key_suggestion_metadata(conn, table_name)
         )
-        targets = await _filter_visible_foreign_key_targets(
-            self.ds, request.actor, database_name, targets
-        )
-        visible_target_tables = {target["fk_table"] for target in targets}
-        current_by_column = {
-            column: current
-            for column, current in current_by_column.items()
-            if current["fk_table"] in visible_target_tables
-        }
 
         columns = []
         options_by_column = {}
@@ -1309,9 +1262,7 @@ class TableAlterView(BaseView):
                     elif operation.op == "set_foreign_keys":
                         foreign_keys = [fk.tuple for fk in args.foreign_keys]
 
-                # Use a savepoint inside execute_write_fn's transaction so
+                with operation_conn:
-                # write_wrapper hooks can still reject and roll back the write.
-                with db_for_write.atomic():
                     for column in add_columns:
                         not_null_default = None
                         if column.not_null:

datasette/views/table_extras.py

@@ -1206,10 +1206,7 @@ class ForeignKeyTablesExtra(Extra):
 
     async def resolve(self, context):
         return await context.foreign_key_tables(
-            context.database_name,
+            context.database_name, context.table_name, context.pk_values
-            context.table_name,
-            context.pk_values,
-            actor=context.request.actor,
         )
 
 

datasette/write_sql.py

@@ -83,22 +83,6 @@ def decision_for_write_sql_operation(
             )
     if operation.operation == "function":
         return IgnoreWriteSqlOperation("SQL function")
-    if (
-        operation.operation == "read"
-        and operation.target_type == "table"
-        and operation.table is not None
-        and operation.table_kind is None
-        and operation.table.lower().startswith("pragma_")
-    ):
-        # Eponymous table-valued PRAGMA functions (e.g. pragma_table_info("secret"))
-        # report a read of the synthetic "pragma_table_info" table, not of the
-        # table passed as an argument. That means a view-table denial on the real
-        # table is never consulted, so these could otherwise be used to read
-        # schema metadata (column names, table lists, ...) for tables the actor
-        # is not allowed to view. Reject them outright in untrusted write SQL,
-        # including inside CREATE VIEW bodies (whose reads are discovered here
-        # via the rolled-back dependency-read analysis above).
-        return UnsupportedWriteSqlOperation(unsupported_message)
     if (
         operation.operation == "read"
         and operation.target_type == "table"

docs/authentication.rst

@@ -158,15 +158,6 @@ Datasette resolves matching rules from most specific to least specific:
 
 This means a resource-level allow can provide an exception to a parent-level deny. It also means that two plugins which disagree at the same level resolve to deny.
 
-For table and view permissions, resource names use SQLite's case-insensitive
-identifier matching: ``Secret``, ``secret`` and ``SECRET`` identify the same
-table. This applies to configuration rules, plugin rules and token restrictions.
-Only ASCII letters are case-insensitive; non-ASCII characters remain distinct.
-Conflicting rules for different spellings of the same name follow the usual
-deny-wins rule at the same scope. Names retain their original spelling in
-resource listings and permission explanations. Database names, stored query
-names and other resource types remain case-sensitive.
-
 .. list-table:: Permission rule examples
    :header-rows: 1
 
@@ -191,18 +182,6 @@ names and other resource types remain case-sensitive.
 
 The built-in public defaults are global allow rules for actions such as ``view-instance``, ``view-database`` and ``view-table``. They follow the same precedence rules as configuration and plugin rules. The ``--default-deny`` option prevents Datasette from contributing those default allow rules.
 
-The built-in ``datasette.default_permissions.sqlite_statistics`` plugin denies
-``view-table`` for ``sqlite_stat1``, ``sqlite_stat2``, ``sqlite_stat3`` and
-``sqlite_stat4``. These table-level denials also apply to root users and take
-precedence over configuration or plugin allow rules at the same scope.
-This controls table access and listings, without changing ``execute-sql`` or
-SQLite's internal use of statistics.
-
-A plugin can replace this policy by unregistering
-``datasette.default_permissions.sqlite_statistics`` through ``datasette.pm``
-and registering its own permission hook. Plugin registration is process-wide:
-replacing this policy affects every Datasette instance in that process.
-
 Datasette performs checks using :ref:`datasette_allowed`, which accepts keyword arguments for ``action``, ``resource`` and an optional ``actor``.
 
 ``resource`` should be an instance of the appropriate ``Resource`` subclass from :mod:`datasette.resources`—for example ``InstanceResource()``, ``DatabaseResource(database="...``)`` or ``TableResource(database="...", table="...")``. This defaults to ``InstanceResource()`` if not specified.
@@ -792,8 +771,6 @@ Datasette defaults to allowing any site visitor to execute their own custom SQL
 
 Access to this ability is controlled by the :ref:`actions_execute_sql` permission.
 
-This permission does not apply to structured table-browsing operations where Datasette constructs the SQL, such as sorting, column filters and :ref:`facets`. Faceting is controlled separately by the :ref:`setting_allow_facet` setting.
-
 The easiest way to disable arbitrary SQL queries is using the :ref:`default_allow_sql setting <setting_default_allow_sql>` when you first start Datasette running.
 
 You can alternatively use an ``"allow_sql"`` block to control who is allowed to execute arbitrary SQL queries.
@@ -1382,12 +1359,6 @@ view-table
 
 Actor is allowed to view a table (or view) page, e.g. https://latest.datasette.io/fixtures/complex_foreign_keys
 
-Derived implementation tables require access to their immediate source: FTS and RTree shadow tables require access to their virtual table, external-content FTS tables require access to their content table, and FTS vocabulary tables (``fts5vocab`` and ``fts4aux``) require access to their FTS table. The derived table's own permission rules also apply.
-
-Access is always denied if the source table is itself derived, or if a vocabulary table's source cannot be identified.
-
-The same rules apply to individual permission checks and table listings, including whether they are private. If a database error prevents dependency discovery, the check or listing fails with an error instead of ignoring the dependencies. Failed discovery results are not cached, so later checks can retry.
-
 ``resource`` - ``datasette.resources.TableResource(database, table)``
     ``database`` is the name of the database (string)
 
@@ -1550,8 +1521,6 @@ execute-sql
 
 Actor is allowed to run arbitrary read-only SQL queries against a specific database using the :ref:`custom SQL query page <pages_custom_sql_queries>`, e.g. https://latest.datasette.io/fixtures/-/query?sql=select+100
 
-This action also controls raw SQL supplied using ``?_where=``. It does not control structured table-browsing features such as :ref:`facets`, which use SQL generated by Datasette and are controlled by :ref:`setting_allow_facet`.
-
 ``resource`` - ``datasette.resources.DatabaseResource(database)``
     ``database`` is the name of the database (string)
 

docs/changelog.rst

@@ -4,121 +4,6 @@
 Changelog
 =========
 
-.. _v1_0_a41:
-
-1.0a41 (2026-09-24)
--------------------
-
-OpenTelemetry support, a new JavaScript API for creating modal dialogs, and several smaller bug fixes.
-
-OpenTelemetry
-~~~~~~~~~~~~~
-
-Datasette now supports `OpenTelemetry <https://opentelemetry.io/>`__ traces and metrics for monitoring application performance. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:issue:`1730`, :issue:`2867`)
-
-- Traces cover HTTP requests, database queries and startup, including time spent waiting for SQL threads and queued writes.
-- Metrics report query latency, time-limit interruptions, SQL thread usage, write queues and open connections.
-- New :ref:`tools for plugin authors <plugin_telemetry>` help plugins add their own traces and metrics, with shared registry classes and pytest helpers.
-
-To collect telemetry, configure an OpenTelemetry SDK and exporter, then run Datasette using ``opentelemetry-instrument``. See :ref:`internals_telemetry` for setup instructions.
-
-Other features
-~~~~~~~~~~~~~~
-
-- New :ref:`DatasetteModal JavaScript API <javascript_plugins_modals>` for plugins to create dialogs with Datasette's shared styles, keyboard behavior and focus handling. Datasette's built-in dialogs use the same API. (:issue:`2790`, :pr:`2948`)
-
-Bug fixes
-~~~~~~~~~
-
-- Table pages now show measured query timings instead of always displaying 1.2ms. (:issue:`2446`)
-- Facet loading now ignores unrelated query parameters such as ``?_facets=x``, instead of returning a 500 error. Thanks, `Peng Boyu <https://github.com/pengboyu-dev>`__. (:pr:`2949`)
-- Foreign key values no longer link to tables that do not exist. Thanks, `Dipak Chaudhari <https://github.com/dchaudhari7177>`__. (:issue:`1515`, :pr:`2952`)
-- Fixed missing punctuation between table and view counts on the homepage, such as ``0 tables1 view``. Thanks, `Dipak Chaudhari <https://github.com/dchaudhari7177>`__. (:issue:`2012`, :pr:`2951`)
-- The sort menu now excludes primary keys that are not included in :ref:`sortable_columns <table_configuration_sortable_columns>`. Thanks, `Sanjay Santhanam <https://github.com/Sanjays2402>`__. (:issue:`1980`, :pr:`2858`)
-
-.. _v1_0_a40:
-
-1.0a40 (2026-09-16)
--------------------
-
-A security fix, a new set of APIs providing background tasks for plugins, an endpoint for counting matching rows, and a collection of bug fixes.
-
-Security fix
-~~~~~~~~~~~~
-
-- Fixed a security issue where a trailing newline in a requested table name could bypass table permissions and expose private rows. Thanks for the report, `dpfkdlemtp <https://github.com/dpfkdlemtp>`__. `GHSA-h547-rmjf-5m2m <https://github.com/simonw/datasette/security/advisories/GHSA-h547-rmjf-5m2m>`__
-
-Background tasks
-~~~~~~~~~~~~~~~~
-
-Datasette plugins can now use **background tasks** to run code independent of the Datasette request/response cycle.
-
-- New :ref:`datasette_add_background_task` API: plugins register supervised, long-lived background work - typically from a ``startup`` hook - and these will be launched after every ``startup`` hook has run. Tasks are cancelled (with a five-second grace period) on shutdown.
-- New ``/-/tasks`` JSON debug endpoint lists every supervised background task and its state, in the style of ``/-/threads``. See :ref:`JsonDataView_tasks`. It requires the ``permissions-debug`` permission.
-- New :ref:`plugin_hook_shutdown` plugin hook, called during graceful shutdown (Ctrl-C, ``SIGTERM``) before background tasks are cancelled and before database connections are closed. It is not called on a hard kill (``SIGKILL``).
-- Plugin ``asgi_wrapper`` middleware now always runs *after* startup has completed.
-- If your plugin uses ``asgi_wrapper`` to start background tasks on the first incoming request, you should migrate to ``datasette.add_background_task()`` instead. `datasette-cron <https://datasette.io/plugins/datasette-cron>`__ and `datasette-enrichments <https://datasette.io/plugins/datasette-enrichments>`__ are being migrated to this pattern.
-
-Other features
-~~~~~~~~~~~~~~
-
-- New :ref:`POST count endpoint <TableCountView>` for counting filtered table rows, now used by the **count all** button. (:issue:`2914`)
-- Datasette now uses `httpx2 <https://httpx2.pydantic.dev/>`__, the Pydantic-maintained continuation of `httpx <https://www.python-httpx.org/>`__, in place of ``httpx``. The public API is the same, but responses returned by :ref:`internals_datasette_client` are now ``httpx2.Response`` objects rather than ``httpx.Response``. Plugins that use ``isinstance()`` checks against ``httpx.Response`` should be updated to use ``httpx2``. **Plugins that use httpx without explicitly depending on it** will need to add an explicit dependency or switch to `httpx2`.
-
-Bug fixes
-~~~~~~~~~
-
-- Column facets now show the remove-filter link for filters using ``column__exact=value``, as well as ``column=value``. (:issue:`1695`)
-- The :ref:`alter-table API <TableAlterView>` now rolls back schema changes when a :ref:`write_wrapper <plugin_hook_write_wrapper>` raises after the write. (:issue:`2924`, :pr:`2925`)
-- The :ref:`extra_template_vars() <plugin_hook_extra_template_vars>` plugin hook can now return a function or awaitable that resolves to ``None`` when no extra variables are needed. (:issue:`2005`)
-- :ref:`request.headers <internals_request>` now supports case-insensitive header lookups, so ``request.headers.get("Content-Type")`` works as well as ``request.headers.get("content-type")``. (:issue:`1861`)
-- CSV endpoints now return plain-text error messages for SQL errors. (:issue:`2129`)
-- The :ref:`render_cell() <plugin_hook_render_cell>` plugin hook now receives an empty ``pks`` list when rendering SQL views in HTML, matching the JSON ``?_extra=render_cell`` behavior. (:issue:`2639`)
-- Numeric comparison filters now correctly handle decimal values, negative numbers and scientific notation when filtering computed columns and SQL views. Thanks, `Rami Abdelrazzaq <https://github.com/RamiNoodle733>`__. (:issue:`1681`, :pr:`2876`)
-- Fixed CSV streaming with ``?_stream=on`` on SQL views repeating the second page of results until the CSV size limit was reached. Thanks, `Ankita Advitot <https://github.com/AnkitaAdvitot>`__. (:issue:`2902`, :pr:`2903`)
-
-.. _v1_0_a39:
-
-1.0a39 (2026-09-10)
--------------------
-
-This alpha release includes security fixes for permissions, SQL construction, HTML rendering, authentication and caching, plus improvements to application startup and write execution.
-
-See `0.65.4 <https://docs.datasette.io/en/stable/changelog.html#v0-65-4>`__ for fixes that have been backported to the stable 0.65.x branch.
-
-The Datasette blog `has more details on these releases <https://datasette.io/blog/2026/september-security-releases/>`__.
-
-Some of the security fixes include:
-
-- Table and view permission checks now take SQLite's case-insensitive names into account. See :ref:`authentication_permissions_explained`.
-- Viewing a full-text search index table now checks you have permission to view the table from which it draws its content.
-- Viewing SQLite statistics tables (``sqlite_stat1`` through ``sqlite_stat4``) is now denied by a default.
-- Table schema display now obeys the ``view-table`` permission.
-- Table filters using ``?_through=`` require permission to view the intermediate table.
-- Foreign-key target and suggestion APIs, incoming foreign-key relationships and their row counts now respect ``view-table`` permission.
-- Row endpoints check permissions before resolving primary keys, to avoid revealing the existence of an otherwise invisible primary key.
-- Improved permission checks for the create-table API. See :ref:`json_api_write`.
-- The write SQL interface now checks ``view-table`` permission for tables referenced by ``CREATE VIEW`` statements.
-- Fixed SQL identifier escaping for column names from untrusted database schemas.
-- Fixed HTML escaping for column names from untrusted database schemas.
-- URL columns now render links only for validated HTTP or HTTPS URLs.
-- Private and personalized dynamic responses now use ``Cache-Control: private, no-store``. Anonymous dynamic responses vary by ``Cookie`` and ``Authorization``.
-- Actor cookies now respect ``expire_after``.
-- Restricted actors can no longer create API tokens.
-- Stored-query create, edit and delete forms now block framing to prevent clickjacking.
-- Configuration secret redaction now matches key names case-insensitively.
-- SQLite extension loading is disabled after extensions supplied using ``--load-extension`` have been loaded.
-
-Other improvements and fixes
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-- :ref:`db.execute_write() <database_execute_write>` now has a default execution time limit of 2,000ms. Plugins can override this using ``time_limit_ms=`` or disable it using ``time_limit_ms=None``. This limit is independent of the ``sql_time_limit_ms`` setting for read queries.
-- Application startup now runs through ASGI lifespan events before requests are accepted, with a first-request fallback for hosts without lifespan support. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:pr:`2887`)
-- ``datasette serve`` now runs startup hooks and Uvicorn on the same event loop, preserving background tasks started by plugins. The minimum Uvicorn version is now 0.29. Thanks, `Alex Garcia <https://github.com/asg017>`__. (:pr:`2886`)
-- Non-blocking writes using ``execute_write_fn(..., block=False)`` now return a distinct task UUID for every call and work correctly with ``num_sql_threads=0``. Thanks, `Zain Dana Harper <https://github.com/HarperZ9>`__. (:issue:`2860`, :issue:`2859`)
-- Dropping a table now disables its full-text search index first. (:issue:`2874`)
-- Fixed ``CREATE VIEW`` SQL analysis on Python 3.10.
-
 .. _v1_0_a38:
 
 1.0a38 (2026-08-06)
@@ -1207,7 +1092,7 @@ Features
 - New ``--nolock`` option for ignoring file locks when opening read-only databases. (:issue:`1744`)
 - Spaces in the database names in URLs are now encoded as ``+`` rather than ``~20``. (:issue:`1701`)
 - ``<Binary: 2427344 bytes>`` is now displayed as ``<Binary: 2,427,344 bytes>`` and is accompanied by tooltip showing "2.3MB". (:issue:`1712`)
-- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image <https://hub.docker.com/r/datasetteproject/datasette>`__ has been upgraded to ``3.10.6-slim-bullseye``.  (:issue:`1768`)
+- The base Docker image used by ``datasette publish cloudrun``, ``datasette package`` and the `official Datasette image <https://hub.docker.com/datasetteproject/datasette>`__ has been upgraded to ``3.10.6-slim-bullseye``.  (:issue:`1768`)
 - Canned writable queries against immutable databases now show a warning message. (:issue:`1728`)
 - ``datasette publish cloudrun`` has a new ``--timeout`` option which can be used to increase the time limit applied by the Google Cloud build environment. Thanks, Tim Sherratt. (:pr:`1717`)
 - ``datasette publish cloudrun`` has new ``--min-instances`` and ``--max-instances`` options. (:issue:`1779`)
@@ -2236,7 +2121,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you
 - Removed obsolete ``?_group_count=col`` feature (:issue:`504`)
 - Improved user interface and documentation for ``datasette publish cloudrun`` (:issue:`608`)
 - Tables with indexes now show the ``CREATE INDEX`` statements on the table page (:issue:`618`)
-- Current version of `uvicorn <https://uvicorn.dev/>`__ is now shown on ``/-/versions``
+- Current version of `uvicorn <https://www.uvicorn.org/>`__ is now shown on ``/-/versions``
 - Python 3.8 is now supported! (:issue:`622`)
 - Python 3.5 is no longer supported.
 
@@ -2287,7 +2172,7 @@ If you are still running Python 3.5 you should stick with ``0.30.2``, which you
 0.29.2 (2019-07-13)
 -------------------
 
-- Bumped `Uvicorn <https://uvicorn.dev/>`__ to 0.8.4, fixing a bug where the query string was not included in the server logs. (:issue:`559`)
+- Bumped `Uvicorn <https://www.uvicorn.org/>`__ to 0.8.4, fixing a bug where the query string was not included in the server logs. (:issue:`559`)
 - Fixed bug where the navigation breadcrumbs were not displayed correctly on the page for a custom query. (:issue:`558`)
 - Fixed bug where custom query names containing unicode characters caused errors.
 
@@ -2309,7 +2194,7 @@ ASGI, new plugin hooks, facet by date and much, much more...
 ASGI
 ~~~~
 
-`ASGI <https://asgi.readthedocs.io/>`__ is the Asynchronous Server Gateway Interface standard. I've been wanting to convert Datasette into an ASGI application for over a year - `Port Datasette to ASGI #272 <https://github.com/simonw/datasette/issues/272>`__ tracks thirteen months of intermittent development - but with Datasette 0.29 the change is finally released. This also means Datasette now runs on top of `Uvicorn <https://uvicorn.dev/>`__ and no longer depends on `Sanic <https://github.com/huge-success/sanic>`__.
+`ASGI <https://asgi.readthedocs.io/>`__ is the Asynchronous Server Gateway Interface standard. I've been wanting to convert Datasette into an ASGI application for over a year - `Port Datasette to ASGI #272 <https://github.com/simonw/datasette/issues/272>`__ tracks thirteen months of intermittent development - but with Datasette 0.29 the change is finally released. This also means Datasette now runs on top of `Uvicorn <https://www.uvicorn.org/>`__ and no longer depends on `Sanic <https://github.com/huge-success/sanic>`__.
 
 I wrote about the significance of this change in `Porting Datasette to ASGI, and Turtles all the way down <https://simonwillison.net/2019/Jun/23/datasette-asgi/>`__.
 
@@ -2723,7 +2608,7 @@ Miscellaneous
   as a string.
 * If you just want an array of the first value of each row, use the new
   ``?_shape=arrayfirst`` option - `example
-  <https://latest.datasette.io/fixtures.json?sql=select+_neighborhood+from+facetable+order+by+pk+limit+101&_shape=arrayfirst>`_.
+  <https://latest.datasette.io/fixtures.json?sql=select+neighborhood+from+facetable+order+by+pk+limit+101&_shape=arrayfirst>`_.
 
 0.22.1 (2018-05-23)
 -------------------

docs/cli-reference.rst

@@ -76,8 +76,6 @@ Or since this is the default command you can run this instead::
 
 Once started you can access it at ``http://localhost:8001``
 
-Use ``--internal PATH`` or the ``DATASETTE_INTERNAL`` environment variable to persist :ref:`Datasette's internal database <internals_internal>` to a SQLite file.
-
 .. [[[cog
     help(["serve", "--help"])
 .. ]]]

docs/contributing.rst

@@ -312,19 +312,6 @@ To update these pages, run the following command::
 
     uv run cog -r docs/*.rst
 
-.. _contributing_documentation_screenshots:
-
-Documentation screenshots
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Screenshots in the documentation are defined in ``docs/shots.yml`` and taken using `shot-scraper <https://shot-scraper.datasette.io/>`__. That file starts a Datasette server that loads JavaScript from ``docs/shots/``, then saves each screenshot as a WebP image in ``docs/images/``.
-
-To take any screenshots that do not exist yet, run::
-
-    just shots
-
-``just docs`` runs this too. Existing images are skipped. To replace a screenshot, delete its image file and run ``just shots`` again.
-
 .. _contributing_template_contexts:
 
 Documented template contexts

docs/custom_templates.rst

@@ -302,14 +302,6 @@ content you can do so by creating a ``row.html`` template like this:
 Note the ``default:row.html`` template name, which ensures Jinja will inherit
 from the default template.
 
-The default ``base.html`` template provides a ``crumbs`` block inside its navigation block. Override ``crumbs`` to customize the breadcrumbs without replacing the rest of the navigation. The imported ``crumbs.nav()`` macro renders Datasette's permission-aware breadcrumbs:
-
-.. code-block:: jinja
-
-    {% block crumbs %}
-    {{ crumbs.nav(request=request, database=database, table=table) }}
-    {% endblock %}
-
 The ``_table.html`` template is included by both the row and the table pages,
 and a list of rows. The default ``_table.html`` template renders them as an
 HTML template and `can be seen here <https://github.com/simonw/datasette/blob/main/datasette/templates/_table.html>`_.

docs/facets.rst

@@ -7,15 +7,13 @@ Datasette facets can be used to add a faceted browse interface to any database t
 With facets, tables are displayed along with a summary showing the most common values in specified columns.
 These values can be selected to further filter the table.
 
-Here's `an example <https://datasette.io/legislators/legislator_terms?_facet=type&_facet=party&_facet=state&_facet_size=10>`__:
+Here's `an example <https://congress-legislators.datasettes.com/legislators/legislator_terms?_facet=type&_facet=party&_facet=state&_facet_size=10>`__:
 
 .. image:: https://raw.githubusercontent.com/simonw/datasette-screenshots/0.62/non-retina/faceting-details.png
    :alt: Screenshot showing facets against a table of congressional legislators. Suggested facets include state_rank and start and end dates, and the displayed facets are state, party and type. Each facet lists values along with a count of rows for each value.
 
 Facets can be specified in two ways: using query string parameters, or in ``metadata.json`` configuration for the table.
 
-Facet queries are generated by Datasette and summarize rows the actor already has permission to view. They do not require the :ref:`actions_execute_sql` permission. Use the :ref:`setting_allow_facet` setting to control whether users can request facets using query string parameters.
-
 Facets in query strings
 -----------------------
 

docs/full_text_search.rst

@@ -19,7 +19,7 @@ The table page and table view API
 
 Table views that support full-text search can be queried using the ``?_search=TERMS`` query string parameter. This will run the search against content from all of the columns that have been included in the index.
 
-Try `searching Datasette ecosystem repositories for "csv" <https://datasette.io/content/repos?_search=csv>`__ to find tools for working with CSV files.
+Try this example: `fara.datasettes.com/fara/FARA_All_ShortForms?_search=manafort <https://fara.datasettes.com/fara/FARA_All_ShortForms?_search=manafort>`__
 
 SQLite full-text search supports wildcards. This means you can easily implement prefix auto-complete by including an asterisk at the end of the search term - for example::
 
@@ -52,11 +52,11 @@ Configuring full-text search for a table or view
 
 If a table has a corresponding FTS table set up using the ``content=`` argument to ``CREATE VIRTUAL TABLE`` shown below, Datasette will detect it automatically and add a search interface to the table page for that table.
 
-You can also manually configure which table should be used for full-text search using table configuration in ``datasette.yaml`` (see :ref:`table_configuration_fts`). You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option.
+You can also manually configure which table should be used for full-text search using query string parameters or table configuration in ``datasette.yaml`` (see :ref:`table_configuration_fts`). You can set the associated FTS table for a specific table and you can also set one for a view - if you do that, the page for that SQL view will offer a search option.
 
-The legacy ``?_fts_table=x`` and ``?_fts_pk=col`` query string parameters are accepted only if they exactly match the configured or automatically detected FTS mapping. They cannot be used to select a different FTS table or primary key. This prevents a public table from being used to probe the contents of a private FTS table.
+Use ``?_fts_table=x`` to over-ride the FTS table for a specific page. If the primary key was something other than ``rowid`` you can use ``?_fts_pk=col`` to set that as well. This is particularly useful for views, for example:
 
-Searching also requires the current actor to have ``view-table`` permission for the FTS table itself, in addition to permission to view the table or view being searched.
+https://latest.datasette.io/fixtures/searchable_view?_fts_table=searchable_fts&_fts_pk=pk
 
 The ``fts_table`` metadata property can be used to specify an associated FTS table. If the primary key column in your table which was used to populate the FTS table is something other than ``rowid``, you can specify the column to use with the ``fts_pk`` property.
 
@@ -120,31 +120,41 @@ Searches using custom SQL
 
 You can include full-text search results in custom SQL queries. The general pattern with SQLite search is to run the search as a sub-select that returns rowid values, then include those rowids in another part of the query.
 
-You can see the syntax for a basic search by running that search on a table page and then clicking "View and edit SQL" to see the underlying SQL. For example, consider this search for `repositories mentioning "csv" <https://datasette.io/content/repos?_search=csv>`_::
+You can see the syntax for a basic search by running that search on a table page and then clicking "View and edit SQL" to see the underlying SQL. For example, consider this search for `manafort is the US FARA database <https://fara.datasettes.com/fara/FARA_All_ShortForms?_search=manafort>`_::
 
-    /content/repos?_search=csv
+    /fara/FARA_All_ShortForms?_search=manafort
 
-The generated SQL selects all columns in the table. This simplified version selects just the repository ID, full name and description, using the same full-text search condition. `Run this query <https://datasette.io/content?sql=select%0A++id%2C%0A++full_name%2C%0A++description%0Afrom%0A++repos%0Awhere%0A++rowid+in+%28%0A++++select%0A++++++rowid%0A++++from%0A++++++repos_fts%0A++++where%0A++++++repos_fts+match+escape_fts%28%3Asearch%29%0A++%29%0Aorder+by%0A++id%0Alimit%0A++101&search=csv>`_ with ``search`` set to ``csv``:
+If you click `View and edit SQL <https://fara.datasettes.com/fara?sql=select%0D%0A++rowid%2C%0D%0A++Short_Form_Termination_Date%2C%0D%0A++Short_Form_Date%2C%0D%0A++Short_Form_Last_Name%2C%0D%0A++Short_Form_First_Name%2C%0D%0A++Registration_Number%2C%0D%0A++Registration_Date%2C%0D%0A++Registrant_Name%2C%0D%0A++Address_1%2C%0D%0A++Address_2%2C%0D%0A++City%2C%0D%0A++State%2C%0D%0A++Zip%0D%0Afrom%0D%0A++FARA_All_ShortForms%0D%0Awhere%0D%0A++rowid+in+%28%0D%0A++++select%0D%0A++++++rowid%0D%0A++++from%0D%0A++++++FARA_All_ShortForms_fts%0D%0A++++where%0D%0A++++++FARA_All_ShortForms_fts+match+escape_fts%28%3Asearch%29%0D%0A++%29%0D%0Aorder+by%0D%0A++rowid%0D%0Alimit%0D%0A++101&search=manafort>`_ you'll see that the underlying SQL looks like this:
 
 .. code-block:: sql
 
     select
-      id,
+      rowid,
-      full_name,
+      Short_Form_Termination_Date,
-      description
+      Short_Form_Date,
+      Short_Form_Last_Name,
+      Short_Form_First_Name,
+      Registration_Number,
+      Registration_Date,
+      Registrant_Name,
+      Address_1,
+      Address_2,
+      City,
+      State,
+      Zip
     from
-      repos
+      FARA_All_ShortForms
     where
       rowid in (
         select
           rowid
         from
-          repos_fts
+          FARA_All_ShortForms_fts
         where
-          repos_fts match escape_fts(:search)
+          FARA_All_ShortForms_fts match escape_fts(:search)
       )
     order by
-      id
+      rowid
     limit
       101
 

docs/index.rst

@@ -64,7 +64,6 @@ Contents
    javascript_plugins
    plugin_hooks
    testing_plugins
-   plugin_telemetry
    internals
    events
    upgrade_guide

docs/internals.rst

@@ -26,7 +26,7 @@ The request object is passed to various plugin hooks. It represents an incoming
     The request scheme - usually ``https`` or ``http``.
 
 ``.headers`` - dictionary (str -> str)
-    A dictionary of incoming HTTP request headers. Header lookups using ``request.headers["Content-Type"]``, ``request.headers.get("Content-Type")`` and ``"Content-Type" in request.headers`` are case-insensitive. Header names are lowercase when iterating over the dictionary.
+    A dictionary of incoming HTTP request headers. Header names have been converted to lowercase.
 
 ``.cookies`` - dictionary (str -> str)
     A dictionary of incoming cookies
@@ -147,7 +147,7 @@ And a class method that can be used to create fake request objects for use in te
 .. _internals_multiparams:
 
 The MultiParams class
----------------------
+=====================
 
 ``request.args`` is a ``MultiParams`` object - a dictionary-like object which provides access to query string parameters that may have multiple values.
 
@@ -177,7 +177,7 @@ Consider the query string ``?foo=1&foo=2&bar=3`` - with two values for ``foo`` a
 .. _internals_formdata:
 
 The FormData class
-------------------
+==================
 
 ``await request.form()`` returns a ``FormData`` object - a dictionary-like object which provides access to form fields and uploaded files. It has a similar interface to ``MultiParams``.
 
@@ -205,7 +205,7 @@ The FormData class
 .. _internals_uploadedfile:
 
 The UploadedFile class
-----------------------
+======================
 
 When parsing multipart form data with ``files=True``, file uploads are returned as ``UploadedFile`` objects with the following properties and methods:
 
@@ -1403,31 +1403,7 @@ Release all resources held by this ``Datasette`` instance. This calls :ref:`data
 
 If a call to ``Database.close()`` on one of the attached databases raises an exception, ``Datasette.close()`` will continue trying to close the remaining databases and will re-raise the first exception after every database has been processed.
 
-When Datasette is being served over ASGI the ``close()`` method is wired up to the lifespan shutdown event, so resources are released cleanly on ``SIGTERM`` / ``SIGINT``. See :ref:`datasette_lifecycle` for where ``close()`` fits into the full startup-to-shutdown sequence.
+When Datasette is being served over ASGI the ``close()`` method is wired up to the lifespan shutdown event, so resources are released cleanly on ``SIGTERM`` / ``SIGINT``.
-
-.. _datasette_add_background_task:
-
-.add_background_task(func, name=None)
--------------------------------------
-
-``func`` - async callable
-    A coroutine function taking one positional argument, the ``Datasette`` instance. Core calls ``await func(datasette)``.
-
-``name`` - string, optional
-    A name for the task, used to identify it in the ``/-/tasks`` introspection endpoint (:ref:`JsonDataView_tasks`) and in log messages. Defaults to ``func.__qualname__``. If the resulting name collides with an already-registered task, a ``-2``, ``-3``, ... suffix is appended.
-
-Registers supervised background work and returns a :ref:`BackgroundTask <BackgroundTask>` handle. Tasks registered during startup launch after all startup hooks finish; tasks registered after launch start immediately.
-
-See :ref:`internals_background_tasks` for examples, launch behavior, task supervision and cancellation.
-
-.. _datasette_start_background_tasks:
-
-await .start_background_tasks()
--------------------------------
-
-Runs startup (if it has not already run) and launches every task registered with :ref:`datasette_add_background_task`.
-
-See :ref:`internals_background_tasks` for when tasks launch automatically, and :ref:`internals_background_tasks_explicit` for examples and startup considerations in tests and headless programs.
 
 .. _datasette_track_event:
 
@@ -1618,32 +1594,32 @@ datasette.client
 
 Plugins can make internal simulated HTTP requests to the Datasette instance within which they are running. This ensures that all of Datasette's external JSON APIs are also available to plugins, while avoiding the overhead of making an external HTTP call to access those APIs.
 
-The ``datasette.client`` object is a wrapper around the `HTTPX2 Python library <https://httpx2.pydantic.dev/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
+The ``datasette.client`` object is a wrapper around the `HTTPX Python library <https://www.python-httpx.org/>`__, providing an async-friendly API that is similar to the widely used `Requests library <https://requests.readthedocs.io/>`__.
 
 It offers the following methods:
 
-``await datasette.client.get(path, **kwargs)`` - returns HTTPX2 Response
+``await datasette.client.get(path, **kwargs)`` - returns HTTPX Response
     Execute an internal GET request against that path.
 
-``await datasette.client.post(path, **kwargs)`` - returns HTTPX2 Response
+``await datasette.client.post(path, **kwargs)`` - returns HTTPX Response
     Execute an internal POST request. Use ``data={"name": "value"}`` to pass form parameters.
 
-``await datasette.client.options(path, **kwargs)`` - returns HTTPX2 Response
+``await datasette.client.options(path, **kwargs)`` - returns HTTPX Response
     Execute an internal OPTIONS request.
 
-``await datasette.client.head(path, **kwargs)`` - returns HTTPX2 Response
+``await datasette.client.head(path, **kwargs)`` - returns HTTPX Response
     Execute an internal HEAD request.
 
-``await datasette.client.put(path, **kwargs)`` - returns HTTPX2 Response
+``await datasette.client.put(path, **kwargs)`` - returns HTTPX Response
     Execute an internal PUT request.
 
-``await datasette.client.patch(path, **kwargs)`` - returns HTTPX2 Response
+``await datasette.client.patch(path, **kwargs)`` - returns HTTPX Response
     Execute an internal PATCH request.
 
-``await datasette.client.delete(path, **kwargs)`` - returns HTTPX2 Response
+``await datasette.client.delete(path, **kwargs)`` - returns HTTPX Response
     Execute an internal DELETE request.
 
-``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX2 Response
+``await datasette.client.request(method, path, **kwargs)`` - returns HTTPX Response
     Execute an internal request with the given HTTP method against that path.
 
 These methods can be used with :ref:`internals_datasette_urls` - for example:
@@ -1660,7 +1636,7 @@ These methods can be used with :ref:`internals_datasette_urls` - for example:
 
 ``datasette.client`` methods automatically take the current :ref:`setting_base_url` setting into account, whether or not you use the ``datasette.urls`` family of methods to construct the path.
 
-For documentation on available ``**kwargs`` options and the shape of the HTTPX2 Response object refer to the `HTTPX2 Async documentation <https://httpx2.pydantic.dev/async/>`__.
+For documentation on available ``**kwargs`` options and the shape of the HTTPX Response object refer to the `HTTPX Async documentation <https://www.python-httpx.org/async/>`__.
 
 .. _internals_datasette_client_actor:
 
@@ -1789,135 +1765,6 @@ Use the ``format="json"`` (or ``"csv"`` or other formats supported by plugins) a
 
 These methods each return a ``datasette.utils.PrefixedUrlString`` object, which is a subclass of the Python ``str`` type. This allows the logic that considers the ``base_url`` setting to detect if that prefix has already been applied to the path.
 
-.. _datasette_lifecycle:
-
-Application lifecycle
-=====================
-
-Datasette guarantees a fixed sequence of events between the moment a ``Datasette`` instance is constructed and the moment its resources are released:
-
-1. ``Datasette(...)`` — the constructor runs synchronously and does not run plugin hooks.
-2. **Startup** — ``await datasette.invoke_startup()`` runs once: it populates the internal database's catalog of table schemas (:ref:`internals_internal`), loads canned queries and column type configuration, then calls every registered :ref:`plugin_hook_startup` hook, in plugin registration order. When Datasette is being served, table-count precomputation for immutable databases runs immediately before this, as part of the same startup sequence.
-3. **Background-task launch** — once *every* ``startup`` hook has finished (not before), every task registered with :ref:`datasette_add_background_task` — by any plugin — is launched. A task registered by one plugin's ``startup`` hook can safely depend on state set up by another plugin's ``startup`` hook, because launch only happens after the whole round of hooks completes.
-4. **Serving** — the instance handles requests (or, for headless or CLI use, does whatever the embedding program does with it).
-5. **Shutdown** — triggered by the ASGI ``lifespan.shutdown`` event (Ctrl-C, ``SIGTERM``) or the end of a ``datasette serve`` process: every :ref:`plugin_hook_shutdown` hook runs first, while background tasks are still alive, so a plugin can tell its own task to wind down gracefully; every still-running background task is then cancelled and given a five-second grace period to actually stop; finally every database connection is released via :ref:`datasette_close`.
-
-.. admonition:: Startup hooks run on the event loop that serves requests
-
-   In every trigger path below, ``startup`` hooks run on the same ``asyncio`` event loop that goes on to accept connections. It is safe to create loop-bound primitives — ``asyncio.Lock``, ``asyncio.Queue``, ``asyncio.Event``, a raw ``asyncio.create_task()`` call — inside a ``startup`` hook, and to register long-lived background work with :ref:`datasette_add_background_task` there.
-
-Three trigger paths
--------------------
-
-- ``datasette serve`` (CLI) — startup and ``uvicorn.Server.serve()`` both run inside a single ``asyncio.run()`` call, so there is exactly one event loop for the whole life of the process.
-- **ASGI lifespan** — ``Datasette.app()`` wires startup and background-task launch into the ``on_startup`` list, and shutdown into the ``on_shutdown`` list, of an internal ``AsgiLifespan`` wrapper. A spec-compliant ASGI server (uvicorn, hypercorn, and others) sends the ``lifespan.startup`` message and waits for ``lifespan.startup.complete`` before delivering any ``http`` or ``websocket`` scope, so startup — including every plugin's own internal-database migrations — is guaranteed to have finished before any request reaches Datasette, including requests seen by plugin :ref:`asgi_wrapper <plugin_asgi_wrapper>` middleware. If a ``startup`` hook raises, ``AsgiLifespan`` sends ``lifespan.startup.failed`` with the exception message instead of hanging or crashing ambiguously, so the host can abort the boot cleanly.
-- **First-request fallback** — an internal ``AsgiRunOnFirstRequest`` wrapper runs the same startup work as a safety net for hosts that never send ASGI lifespan events at all: some ASGI mounts, a bare ``app()`` embedded inside another framework, and :ref:`datasette.client <internals_datasette_client>` / test clients, which drive requests directly over ``httpx2.ASGITransport`` without ever emitting ``lifespan.startup``. It runs startup exactly once, the first time any non-lifespan scope arrives, guarded by a lock so that concurrent early requests can't run it twice.
-
-All three paths call the same idempotent internal methods, so it is safe for more than one of them to fire — lifespan startup completing and then a first request arriving afterwards is a no-op the second time. A host that never sends lifespan events and never goes through the CLI degrades to first-request timing: startup runs on the first request instead of before it, exactly as Datasette always worked prior to this lifecycle guarantee. This is a deliberate fallback rather than a regression — see :ref:`internals_background_tasks` for how to opt out of launching background tasks (the ``--get`` CLI path) or drive startup and launch explicitly (tests, headless embedders).
-
-.. _internals_background_tasks:
-
-Background tasks
-================
-
-Datasette can supervise long-lived background work for plugins, such as polling for updates. Register work using :ref:`datasette_add_background_task` and use the returned :ref:`BackgroundTask <BackgroundTask>` handle to inspect or cancel it. See :ref:`datasette_lifecycle` for how background tasks fit into the application's startup and shutdown sequence.
-
-Registering tasks
------------------
-
-Use :ref:`datasette_add_background_task` to register an async callable, typically from a :ref:`plugin_hook_startup` hook. The callable takes one argument, the ``Datasette`` instance. Tasks can also be registered later, including from a request handler.
-
-Registration is separate from launch. Calling this from a ``startup`` hook — the common case — buffers the task; core launches every registered task once *all* ``startup`` hooks have completed, as described in :ref:`datasette_lifecycle`. Calling it after launch has already happened — for example from a request handler, to start a per-job task dynamically — starts the task immediately instead.
-
-.. code-block:: python
-
-    import asyncio
-    from datasette import hookimpl
-
-
-    async def poll_for_updates(datasette):
-        while True:
-            await do_one_poll(datasette)
-            await asyncio.sleep(60)
-
-
-    @hookimpl
-    def startup(datasette):
-        datasette.add_background_task(
-            poll_for_updates, name="my-plugin-poller"
-        )
-
-Datasette supervises each registered task:
-
-- Keeps the task alive.
-- Logs exceptions other than ``asyncio.CancelledError``, with their tracebacks, to the ``datasette.background_tasks`` logger. The exception is recorded on the handle's ``.exception``, and its ``.state`` becomes ``crashed``.
-- Cancels running tasks during shutdown and gives them five seconds to stop. See :ref:`datasette_lifecycle`.
-
-.. _internals_background_tasks_launch:
-
-Launch matrix
--------------
-
-Whether registered tasks actually launch depends on how the instance is being run:
-
-.. list-table::
-   :header-rows: 1
-
-   * - Trigger
-     - Launches registered tasks?
-   * - ASGI lifespan (real server deployments)
-     - Yes, after ``lifespan.startup`` completes
-   * - First-request fallback (lifespan-less hosts)
-     - Yes, on the first request — parity with the lifespan case
-   * - ``datasette serve --get``
-     - Never
-   * - Tests / headless embedders
-     - Only if you call :ref:`datasette_start_background_tasks` explicitly
-
-``datasette --get`` never launches background tasks, even though its one-shot request flows through the same first-request fallback as everything else: it sets an internal flag before making that request specifically to suppress the launch, since a one-shot CLI invocation has no server loop left running afterwards to keep any launched tasks alive.
-
-.. _internals_background_tasks_explicit:
-
-Starting tasks explicitly
--------------------------
-
-Call :ref:`datasette_start_background_tasks` to run startup (if it has not already run) and launch every task registered with :ref:`datasette_add_background_task`. This is the explicit equivalent of what happens automatically via ASGI lifespan or the first-request fallback in a served deployment — the entry point for tests and headless embedders (a cron-style CLI command that wants supervised background work without running a server) that need background tasks without going through either of those paths.
-
-.. code-block:: python
-
-    datasette = Datasette(memory=True)
-    await datasette.start_background_tasks()
-
-.. _BackgroundTask:
-
-BackgroundTask objects
-----------------------
-
-:ref:`datasette_add_background_task` returns a ``BackgroundTask`` handle with the following attributes:
-
-``.name`` - string
-    The task's (unique) name.
-
-``.state`` - string
-    One of ``registered`` (added but not yet launched), ``running``, ``completed`` (returned cleanly), ``crashed`` (raised an exception) or ``cancelled``.
-
-``.task`` - ``asyncio.Task`` or ``None``
-    The underlying ``asyncio.Task``, once launched. ``None`` while still ``registered``.
-
-``.exception`` - ``BaseException`` or ``None``
-    The exception that crashed the task, if ``.state`` is ``crashed``.
-
-``.started_at`` - string or ``None``
-    ISO 8601 UTC timestamp of when the task was launched.
-
-``.function`` - string
-    The callable's dotted module and qualified name, for example ``my_plugin.jobs.poll_for_updates``.
-
-``.cancel()``
-    Cancel the task. If it has already launched, this cancels the underlying ``asyncio.Task`` — ``.state`` becomes ``cancelled`` once the cancellation is observed. If it has not launched yet, it is removed from the queue so it never runs.
-
-This is also the shape of each entry returned by the ``/-/tasks`` JSON introspection endpoint — see :ref:`JsonDataView_tasks`.
-
 .. _internals_permission_classes:
 
 Permission classes and utilities
@@ -2174,12 +2021,10 @@ Example usage:
 
     version = await db.execute_fn(get_version)
 
-The call is traced as a ``db.query`` OpenTelemetry span carrying ``datasette.callback`` (the function's qualified name) rather than ``db.query.text``, since the SQL is whatever the function chooses to run - see :ref:`internals_telemetry`. Passing a named function gives the span a readable identity; a lambda reports ``<lambda>``.
-
 .. _database_execute_write:
 
-await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True, time_limit_ms=2000)
+await db.execute_write(sql, params=None, block=True, request=None, return_all=False, returning_limit=10, transaction=True)
-----------------------------------------------------------------------------------------------------------------------------------------------
+--------------------------------------------------------------------------------------------------------------------------
 
 SQLite only allows one database connection to write at a time. Datasette handles this for you by maintaining a queue of writes to be executed against a given database. Plugins can submit write operations to this queue and they will be executed in the order in which they are received.
 
@@ -2218,13 +2063,6 @@ Each call to ``execute_write()`` will be executed inside a transaction. Pass
 ``transaction=False`` for statements such as ``VACUUM`` that cannot run inside
 a transaction.
 
-Write statements have a default time limit of 2,000ms. Pass a different value
-using ``time_limit_ms=`` or use ``time_limit_ms=None`` to allow the statement to
-run without a time limit.
-
-This write limit is independent of the ``sql_time_limit_ms`` setting used for
-read queries. Changing that setting does not change the default write limit.
-
 .. _database_execute_write_script:
 
 await db.execute_write_script(sql, block=True)
@@ -2259,8 +2097,6 @@ This method works like ``.execute_write()``, but instead of a SQL statement you
 
 The function can then perform multiple actions, safe in the knowledge that it has exclusive access to the single writable connection for as long as it is executing.
 
-Like ``execute_fn()``, the call is traced as a ``db.query`` OpenTelemetry span carrying ``datasette.callback`` rather than ``db.query.text``, above the write-queue spans - see :ref:`internals_telemetry`. A named function gives the span a readable identity; a lambda reports ``<lambda>``.
-
 .. warning::
 
     ``fn`` needs to be a regular function, not an ``async def`` function.
@@ -2317,25 +2153,7 @@ The value returned from ``await database.execute_write_fn(...)`` will be the ret
 
 If your function raises an exception that exception will be propagated up to the ``await`` line.
 
-By default Datasette manages the transaction. For nested transactions, use `sqlite_utils.Database(conn).atomic() <https://sqlite-utils.datasette.io/en/stable/python-api.html#grouping-changes-with-db-atomic>`__. Pass ``transaction=False`` to manage transactions yourself.
+By default your function will be executed inside a transaction. You can pass ``transaction=False`` to disable this behavior, though if you do that you should be careful to manually apply transactions - ideally using the ``with conn:`` pattern, or you may see ``OperationalError: database table is locked`` errors.
-
-For example, archive an article and record the change in an audit log:
-
-.. code-block:: python
-
-    import sqlite_utils
-
-
-    def archive_article(conn):
-        db = sqlite_utils.Database(conn)
-        with db.atomic():
-            db["articles"].update(1, {"archived": True})
-            db["audit_log"].insert(
-                {"article_id": 1, "action": "archive"}
-            )
-
-
-    await database.execute_write_fn(archive_article)
 
 If you specify ``block=False`` the method becomes fire-and-forget, queueing your function to be executed and then allowing your code after the call to ``.execute_write_fn()`` to continue running while the underlying thread waits for an opportunity to run your function. A UUID representing the queued task will be returned. Any exceptions in your code will be silently swallowed.
 
@@ -2495,259 +2313,6 @@ The ``Database`` class also provides properties and methods for introspecting th
           }
         }
 
-.. _internals_telemetry:
-
-OpenTelemetry
-=============
-
-Datasette uses the `opentelemetry-api <https://pypi.org/project/opentelemetry-api/>`__ library to provide `OpenTelemetry <https://opentelemetry.io>`__ traces and metrics for Datasette applications.
-
-Datasette emits telemetry under the ``datasette`` instrumentation scope. To enable tracing, run Datasette under the ``opentelemetry-instrument`` agent.
-
-Plugins can emit their own spans and metrics alongside these, using the same registry classes and test helpers core uses - see :ref:`plugin_telemetry`.
-
-.. _internals_telemetry_turning_on:
-
-Turning tracing on
-------------------
-
-Install an OpenTelemetry SDK, an exporter and the instrumentation agent, then launch Datasette through ``opentelemetry-instrument``:
-
-.. code-block:: bash
-
-    pip install opentelemetry-distro opentelemetry-exporter-otlp
-
-    OTEL_SERVICE_NAME=datasette \
-    OTEL_METRICS_EXPORTER=console \
-    OTEL_LOGS_EXPORTER=console \
-    OTEL_TRACES_EXPORTER=console \
-      opentelemetry-instrument datasette mydb.db
-
-Or using ``uv run``:
-
-.. code-block:: bash
-
-    OTEL_SERVICE_NAME=datasette \
-    OTEL_METRICS_EXPORTER=console \
-    OTEL_LOGS_EXPORTER=console \
-    OTEL_TRACES_EXPORTER=console \
-    uv run \
-      --with opentelemetry-distro \
-      --with opentelemetry-exporter-otlp \
-      opentelemetry-instrument datasette mydb.db
-
-This will output pretty-printed JSON telemetry to your console, representing requests and database queries executed by Datasette.
-
-To use an exporter endpoint, set ``OTEL_EXPORTER_OTLP_ENDPOINT`` to a URL, set ``OTEL_TRACES_EXPORTER`` to ``otlp``, and set the other exporters to ``none``:
-
-.. code-block:: bash
-
-    OTEL_SERVICE_NAME=datasette \
-    OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 \
-    OTEL_METRICS_EXPORTER=none \
-    OTEL_LOGS_EXPORTER=none \
-    OTEL_TRACES_EXPORTER=otlp \
-      opentelemetry-instrument datasette mydb.db
-
-On macOS one easy option for a port 4317 OTLP endpoint is `otel-tui <https://github.com/ymtdzzz/otel-tui>`__:
-
-.. code-block:: bash
-
-    brew install ymtdzzz/tap/otel-tui
-    otel-tui
-
-Traces sent to port 4317 by Datasette will now display in a TUI in your terminal.
-
-A few things catch people out:
-
-- **You must use "opentelemetry-instrument datasette"**. Running just ``OTEL_TRACES_EXPORTER=console datasette mydb.db`` produces no telemetry.
-- **Spans do not appear immediately.** The SDK's default ``BatchSpanProcessor`` flushes on a timer, every 5 seconds. Either wait, or stop the process - shutdown triggers a final flush - or set ``OTEL_BSP_SCHEDULE_DELAY=1000`` while you are experimenting.
-- **Always set** ``OTEL_SERVICE_NAME``. Without it the SDK's default resource reports a ``service.name`` of ``unknown_service``, and your traces will be filed under that instead of under a name you can search for.
-
-.. _internals_telemetry_requests:
-
-Span reference
---------------
-
-Datasette emits six spans. One covers the HTTP request, and is the root everything else raised while serving that request hangs from. Four describe the database layer - one per query, one for the work that query does inside a SQL worker thread, and two more for the write queue. The sixth covers startup. Attribute names use the ``datasette.*`` prefix for Datasette-specific data, alongside standard OpenTelemetry attributes such as ``db.system``.
-
-A request to a table page produces a span named, in full::
-
-    GET /(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)(\.(?P<format>\w+))?$
-
-.. [[[cog
-    from telemetry_doc import spans
-    spans(cog)
-.. ]]]
-
-``{http.request.method} {http.route}``
-    One span per HTTP request, containing spans from plugin middleware and database operations. Named for the HTTP method and matched route, or just the method if no route matches. Incoming ``traceparent`` headers are extracted using the global propagator to continue the caller's trace. Incoming ``baggage`` is not propagated into plugin or downstream context in this release. Set ``OTEL_PROPAGATORS=none`` to disable extraction. For public instances, strip trace context headers at your proxy if callers should not supply trace context.
-
-    Kind: ``SERVER``.
-
-    Attributes:
-
-    - ``http.request.method`` - The HTTP request method. Methods outside the nine defined by RFC 9110 and RFC 5789 are recorded as ``_OTHER``.
-    - ``http.route`` *(optional)* - The regular expression for the matched route, for example ``/(?P<database>[^\/\.]+)/(?P<table>[^\/\.]+)(\.(?P<format>\w+))?$`` for a table page. Use this attribute to group requests by route. Omitted when no route matches.
-    - ``url.path`` - The URL path, excluding the query string.
-    - ``url.scheme`` - ``http`` or ``https``.
-    - ``server.address`` *(optional)* - The ``Host`` header, including any ``:port`` suffix. This value is supplied by the client.
-    - ``user_agent.original`` *(optional)* - The ``User-Agent`` header, verbatim. Omitted if the client sent none.
-    - ``http.response.status_code`` *(optional)* - The HTTP response status code. Omitted if no response was started.
-    - ``error.type`` *(optional)* - The exception class name for a failed operation. On HTTP spans, also set to the status code as a string for 5xx responses. A 4xx response alone does not set this attribute or an error status.
-    - ``datasette.internal_client`` *(optional)* - ``True`` for requests made through ``datasette.client``. Calls made inside another request produce a nested ``SERVER`` span. Filter on this attribute to exclude internal requests from request counts. Omitted for requests received over the network.
-
-``db.query``
-    A SQL operation, including time spent queued for a worker thread. For ``block=False`` writes, the span ends after the write is queued. Callback methods record ``datasette.callback`` in place of ``db.query.text``.
-
-    Kind: ``CLIENT``.
-
-    Attributes:
-
-    - ``db.system`` - Always ``sqlite``.
-    - ``db.namespace`` - Name of the database being queried.
-    - ``db.query.text`` *(optional)* - The SQL, truncated to 2048 characters. Bound parameter values are not recorded. For callback methods, ``datasette.callback`` is recorded instead.
-    - ``datasette.callback`` *(optional)* - The qualified name of the Python callable passed to ``execute_fn()``, ``execute_write_fn()`` or ``execute_isolated_fn()``, for example ``TableInsertView.post.<locals>.insert_or_upsert_rows``. Set instead of ``db.query.text``. Lambdas appear as ``<lambda>``; use a named function for a more descriptive span.
-    - ``db.operation.name`` *(optional)* - The statement's leading keyword, such as ``SELECT``, ``INSERT`` or ``CREATE``, if it matches the supported allowlist. Statements beginning with a common table expression report ``WITH``. Omitted for unrecognized keywords and ``execute_write_script()``.
-    - ``datasette.param_count`` *(optional)* - Number of bound parameters. Recorded instead of the values themselves.
-    - ``datasette.param_sets`` *(optional)* - Number of parameter sets consumed by ``execute_write_many()``. The parameter values are not recorded.
-    - ``datasette.time_limit_ms`` *(optional)* - Time limit applied to the read query, in milliseconds: :ref:`setting_sql_time_limit_ms` or a shorter ``custom_time_limit``.
-    - ``datasette.rows_returned`` *(optional)* - Number of rows returned by a successful read query.
-    - ``datasette.truncated`` *(optional)* - True if the result was cut short by :ref:`setting_max_returned_rows`.
-    - ``datasette.interrupted`` *(optional)* - True if the query exceeded its time limit. The span status is set to ``ERROR`` unless the caller used a ``custom_time_limit`` shorter than :ref:`setting_sql_time_limit_ms`, in which case the status is left unset.
-    - ``datasette.sql_error_suppressed`` *(optional)* - True for a non-timeout SQL error with ``log_sql_errors=False``. The exception is still raised, but the span status is left unset.
-    - ``datasette.executescript`` *(optional)* - True for ``execute_write_script()``, which runs multiple statements.
-    - ``datasette.executemany`` *(optional)* - True for ``execute_write_many()``, which runs one statement against many parameter sets.
-
-``db.query.execute``
-    The read executing inside a SQL worker thread. Child of ``db.query``; the gap between the two is time spent waiting for a thread.
-
-    No attributes.
-
-``db.write.queue_wait``
-    Time a write spent waiting in its database's write queue. For ``block=True``, this is a child of ``db.query``. For ``block=False``, it is a root span linked to the span that queued the write, since the write can outlive that request.
-
-    No attributes.
-
-``db.write.execute``
-    The write executing on the write thread. For ``block=True``, this is a child of ``db.query``. For ``block=False``, it is a root span linked to the span that queued the write.
-
-    Attributes:
-
-    - ``datasette.isolated_connection`` - True if the write ran on its own connection rather than the shared write connection.
-    - ``datasette.transaction`` - False for statements such as ``VACUUM`` that cannot run inside a transaction.
-
-``datasette.startup``
-    Startup work performed by ``invoke_startup()``, including registration hooks, schema catalog updates, saved queries, column type configuration and the ``startup`` hook. Runs during instance startup, either before serving requests or as part of the first request.
-
-    No attributes.
-
-.. [[[end]]]
-
-.. _internals_telemetry_metrics:
-
-Metric reference
-----------------
-
-Spans describe events; metrics describe levels and rates. Metrics can be used to answer questions like "Am I saturating my :ref:`setting_num_sql_threads` threads right now?". Trace sampling drops a portion of traces but does not drop any metrics.
-
-Datasette configures duration histograms in **seconds**. OpenTelemetry's default boundaries are tuned for milliseconds but these would file every SQLite query into a single bucket, making quantile queries meaningless.
-
-This reference is also generated from ``datasette/telemetry_registry.py``:
-
-.. [[[cog
-    from telemetry_doc import metrics
-    metrics(cog)
-.. ]]]
-
-``db.client.operation.duration``
-    Histogram, unit ``s``. Duration of a SQL operation, including callback-based calls such as ``execute_fn()``. For ``block=False`` writes, measures enqueue time.
-
-    Bucket boundaries: ``0.0001``, ``0.0005``, ``0.001``, ``0.005``, ``0.01``, ``0.05``, ``0.1``, ``0.5``, ``1``, ``5``, ``10``.
-
-    Attributes:
-
-    - ``db.system`` - Always ``sqlite``.
-    - ``db.namespace`` - Name of the database being queried.
-    - ``datasette.operation`` - Whether the operation was a read or a write. One of: ``read``, ``write``.
-    - ``error.type`` *(optional)* - The exception class name for a failed operation. On HTTP spans, also set to the status code as a string for 5xx responses. A 4xx response alone does not set this attribute or an error status.
-
-``datasette.write.queue_wait``
-    Histogram, unit ``s``. Time each write waited in its database's write queue.
-
-    Bucket boundaries: ``0.0001``, ``0.0005``, ``0.001``, ``0.005``, ``0.01``, ``0.05``, ``0.1``, ``0.5``, ``1``, ``5``, ``10``.
-
-    Attributes:
-
-    - ``db.namespace`` - Name of the database being queried.
-
-``datasette.sql.queries.interrupted``
-    Counter, unit ``{query}``. Queries cancelled for exceeding :ref:`setting_sql_time_limit_ms`. A rising rate can indicate that queries need optimization or a higher time limit. Caller-selected timeouts shorter than this limit, such as those used for facet suggestion, are excluded.
-
-    Attributes:
-
-    - ``db.namespace`` - Name of the database being queried.
-
-``datasette.sql.threads.limit``
-    Observable gauge, unit ``{thread}``. Maximum concurrent read queries, configured by :ref:`setting_num_sql_threads`. Not reported when ``num_sql_threads`` is ``0``.
-
-    No attributes.
-
-``datasette.sql.threads.queue_depth``
-    Observable gauge, unit ``{query}``. Read queries waiting for a free SQL thread. Sustained values above zero indicate a saturated read pool.
-
-    No attributes.
-
-``datasette.sql.queries.pending``
-    Observable gauge, unit ``{query}``. Read queries submitted to the pool and not yet complete. Sum across databases and compare with ``datasette.sql.threads.limit`` to assess pool usage.
-
-    Attributes:
-
-    - ``db.namespace`` - Name of the database being queried.
-
-``datasette.write.queue_depth``
-    Observable gauge, unit ``{write}``. Writes waiting for a database's single write thread. Increasing ``num_sql_threads`` does not increase write concurrency. Not reported for databases that have never been written to.
-
-    Attributes:
-
-    - ``db.namespace`` - Name of the database being queried.
-
-``datasette.connections.open``
-    Observable gauge, unit ``{connection}``. Open SQLite connections managed by Datasette.
-
-    Attributes:
-
-    - ``db.namespace`` - Name of the database being queried.
-
-.. [[[end]]]
-
-Exemplars
-~~~~~~~~~
-
-An OpenTelemetry `exemplar <https://opentelemetry.io/docs/specs/otel/metrics/data-model/#exemplars>`__ attaches a trace ID and span ID to one sample backing a histogram measurement. Where a spike in ``db.client.operation.duration`` alone tells you "queries were slow sometime in this minute", the exemplar attached to one of the samples in that spike gives you the trace ID of an actual slow query to open:
-
-.. code-block:: text
-
-    db.client.operation.duration count=4
-      exemplars: 4
-        value=0.001564s trace_id=ddfaf45fd4e14913497d7efeac95f381 span_id=fd5792bdbb01e533
-        value=0.006320s trace_id=34aea775ade11a3c5f716695731000fe span_id=25ed9e29dd84dbee
-        value=0.045253s trace_id=a65cb58d1460a179f0d04046ff51ed0d span_id=7f34d6378c85d062
-        value=0.305240s trace_id=6089f4c515c221c0ca7bb53667b37ac8 span_id=0516f4a6641eaa0b
-
-.. _internals_telemetry_privacy:
-
-Privacy and safety
-------------------
-
-Datasette does not configure a telemetry exporter itself. If you enable one, traces may contain sensitive information:
-
-- **SQL text is truncated to 2048 characters.** Literal values in that text are retained. Bound SQL parameter values are not added as attributes; ``datasette.param_count`` records only their count.
-- **Request spans include URL paths, host names and User-Agent headers.** Paths can include identifying values such as row primary keys. Core does not add actor identifiers, cookies, authorization headers, client IP addresses or a ``url.query`` attribute.
-- **Exception messages and tracebacks may be recorded.** These can contain data from requests or database operations.
-
-Review what your application and plugins record before exporting telemetry to an external service. Restrict access to exported data and configure redaction or filtering where needed.
-
 .. _internals_csrf:
 
 CSRF protection
@@ -2769,20 +2334,7 @@ No token, cookie, or hidden form field is needed. Any ``<form method="POST">`` i
 Datasette's internal database
 =============================
 
-Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown.
+Datasette maintains an "internal" SQLite database used for configuration, caching, and storage. Plugins can store configuration, settings, and other data inside this database. By default, Datasette will use a temporary in-memory SQLite database as the internal database, which is created at startup and destroyed at shutdown. Users of Datasette can optionally pass in a ``--internal`` flag to specify the path to a SQLite database to use as the internal database, which will persist internal data across Datasette instances.
-
-To persist internal data across Datasette instances, use the ``--internal`` option to specify the path to a SQLite database:
-
-.. code-block:: bash
-
-    datasette mydatabase.db --internal internal.db
-
-You can also set the ``DATASETTE_INTERNAL`` environment variable to specify this path without passing ``--internal`` each time:
-
-.. code-block:: bash
-
-    export DATASETTE_INTERNAL=/path/to/internal.db
-    datasette mydatabase.db
 
 Datasette maintains tables called ``catalog_databases``, ``catalog_tables``, ``catalog_views``, ``catalog_columns``, ``catalog_indexes``, ``catalog_foreign_keys`` with details of the attached databases and their schemas. These tables should not be considered a stable API - they may change between Datasette releases.
 
@@ -3071,12 +2623,12 @@ This example uses trace to record the start, end and duration of any HTTP GET re
 .. code-block:: python
 
     from datasette.tracer import trace
-    import httpx2
+    import httpx
 
 
     async def fetch_url(url):
         with trace("fetch-url", url=url):
-            async with httpx2.AsyncClient() as client:
+            async with httpx.AsyncClient() as client:
                 return await client.get(url)
 
 .. _internals_tracer_trace_child_tasks:

docs/introspection.rst

@@ -9,7 +9,7 @@ Each of these pages can be viewed in your browser. Add ``.json`` to the URL to g
 
 JSON responses that return an object include an ``"ok": true`` key, consistent with the rest of the :ref:`JSON API <json_api>`.
 
-The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise <json_api_stability>`, with the exception of the debug endpoints ``/-/threads``, ``/-/tasks`` and ``/-/actions``, whose shapes may change in future releases.
+The introspection endpoints documented on this page are covered by the :ref:`JSON API stability promise <json_api_stability>`, with the exception of the debug endpoints ``/-/threads`` and ``/-/actions``, whose shapes may change in future releases.
 
 .. _JsonDataView_metadata:
 
@@ -76,7 +76,7 @@ Shows the version of Datasette, Python and SQLite. `Versions example <https://la
 /-/plugins
 ----------
 
-Shows a list of currently installed plugins and their versions. `Plugins example <https://datasette.io/-/plugins>`_:
+Shows a list of currently installed plugins and their versions. `Plugins example <https://san-francisco.datasettes.com/-/plugins>`_:
 
 .. code-block:: json
 
@@ -278,42 +278,6 @@ Shows details of threads and ``asyncio`` tasks. This endpoint requires the ``per
         ]
     }
 
-.. _JsonDataView_tasks:
-
-/-/tasks
---------
-
-Shows the state of every supervised background task registered with :ref:`datasette.add_background_task() <datasette_add_background_task>`; see also :ref:`BackgroundTask <BackgroundTask>` for what each field below means, and :ref:`datasette_lifecycle` for when tasks are launched. This endpoint requires the ``permissions-debug`` permission, since a crashed task's ``exception`` field can reveal internals such as file paths or query text:
-
-.. code-block:: json
-
-    {
-        "ok": true,
-        "tasks": [
-            {
-                "name": "my_plugin.poll_for_updates",
-                "state": "running",
-                "function": "my_plugin.poll_for_updates",
-                "started_at": "2026-07-30T12:00:00+00:00",
-                "exception": null
-            },
-            {
-                "name": "my_plugin.broken_task",
-                "state": "crashed",
-                "function": "my_plugin.broken_task",
-                "started_at": "2026-07-30T12:00:00+00:00",
-                "exception": "ValueError('something went wrong')"
-            }
-        ],
-        "launched": true
-    }
-
-Each entry's ``function`` identifies the callable by its dotted module and qualified name.
-
-Each entry's ``state`` is one of ``registered`` (added but not yet launched), ``running``, ``completed``, ``crashed`` or ``cancelled``. ``exception`` is a one-line ``repr()`` of the exception for a ``crashed`` task, or ``null`` otherwise - the full traceback is written to the ``datasette.background_tasks`` logger instead, to keep this payload skimmable.
-
-The top-level ``launched`` flag reports whether the instance has run its one-time background task launch (after ``startup`` hooks finish, or via lifespan/first-request/:ref:`start_background_tasks() <datasette_start_background_tasks>`). It distinguishes "no tasks have been registered" (``tasks`` is empty either way) from "tasks are registered but nothing has armed the launch yet" (``launched`` is ``false`` and every task's ``state`` is still ``registered``) - useful when debugging a host that never triggers Datasette's lifespan events.
-
 .. _JsonDataView_actor:
 
 /-/actor

docs/javascript_plugins.rst

@@ -1,7 +1,7 @@
 .. _javascript_plugins:
 
-JavaScript in plugins
+JavaScript plugins
-=====================
+==================
 
 Datasette can run custom JavaScript in several different ways:
 
@@ -474,136 +474,6 @@ Custom fields are responsible for preserving the accessibility of the form:
 
 Plugins should not submit the row themselves from inside ``makeColumnField()`` controls. Datasette owns the insert/edit dialog lifecycle, form submission, API call, error handling and row refresh.
 
-.. _javascript_plugins_modals:
-
-Reusable modal dialogs
-----------------------
-
-Plugins can use ``DatasetteModal`` to create dialogs with the same appearance and keyboard behavior as Datasette's built-in dialogs. The component provides a native modal dialog, shared styles, Escape and backdrop dismissal, busy-state dismissal guards and focus restoration.
-
-Creating a dialog
-~~~~~~~~~~~~~~~~~
-
-``DatasetteModal.create()`` returns a detached ``<datasette-modal>`` element containing a native ``<dialog>``. Access that native element through ``modal.dialog``. Populate its content before appending the wrapper to the page, then call ``modal.show()`` to open it.
-
-This example uses the :ref:`datasette_init event <javascript_datasette_init>` to add a button that opens a dialog:
-
-.. literalinclude:: shots/modal-example.js
-   :language: javascript
-
-Clicking that button opens this dialog:
-
-.. only:: not latex
-
-    .. image:: images/modal-example.webp
-       :width: 584px
-       :alt: A dialog titled Example dialog, with the text "This dialog uses Datasette's shared styles and keyboard behavior." and a Close button in the footer, shown in front of a dimmed Datasette page
-
-Opening and closing
-~~~~~~~~~~~~~~~~~~~
-
-``modal.show(options)``
-    Opens the native dialog using ``showModal()``. ``options`` is an optional object with these optional properties:
-
-    - ``returnFocusTo`` (DOM element): Focus returns to this element when the dialog closes. Defaults to the element with keyboard focus immediately before the dialog opens.
-    - ``initialFocus`` (DOM element or function): An element inside the dialog whose ``focus()`` method will be called, or a function called with no arguments that moves focus itself.
-
-    Use this to focus on an input field when the dialog opens.
-
-``modal.close(options)``
-    Closes the dialog directly. ``options`` is an optional object with one optional property:
-
-    - ``restoreFocus`` (boolean): Whether closing returns focus to the element recorded by ``show()``. Defaults to ``true``.
-
-``modal.requestClose(source)``
-    Alternative to ``.close()`` that requests dismissal through the busy-state and ``beforeClose`` guards described below. Returns ``true`` if it closes the dialog, or ``false`` if the dialog is already closed or a guard prevents dismissal. Close and Cancel buttons should use this method.
-
-    ``source`` is an optional string that is passed to ``beforeClose`` and identifies what requested dismissal. Datasette supplies ``"escape"`` for the Escape key or a native cancel event and ``"backdrop"`` for a click outside the dialog. ``source`` defaults to ``"cancel"``.
-
-Listen for the native dialog's ``close`` event to clean up resources such as pending requests or custom fields:
-
-.. code-block:: javascript
-
-    modal.dialog.addEventListener("close", () => {
-        // Clean up content-specific resources here.
-    });
-
-If the dialog is no longer needed, remove the wrapper with ``modal.remove()``.
-
-Dismissal guards and busy state
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can set ``modal.beforeClose`` to a synchronous function that receives the ``source`` string described above and returns ``false`` in order to keep the dialog open.
-
-Use ``source`` to decide what to do. This example prompts the user to ask if they want to discard unsaved changes - for example if they click outside the modal or hit Escape - but doesn't prompt them if they clicked a button like the Close one above that sets the ``source`` string to ``cancel``.
-
-.. code-block:: javascript
-
-    modal.beforeClose = (source) => {
-        if (source === "cancel") return true;
-        return confirm("Discard unsaved changes?");
-    };
-
-
-Set ``modal.busy = true`` while saving to prevent user dismissal.  While busy, ``requestClose()`` returns ``false`` without calling ``beforeClose``.
-
-If an operation fails, set ``modal.busy = false`` so the user can retry or close the dialog. A successful operation can call ``modal.close()`` even while busy.
-
-.. _javascript_plugins_modal_classes:
-
-Shared CSS classes
-~~~~~~~~~~~~~~~~~~
-
-The classes in the example above provide built-in styling. This dialog uses every class listed below, including a ``modal-meta`` count in the header and ``footer-info`` text next to ``modal-btn-ghost`` and ``modal-btn-primary`` buttons in the footer:
-
-.. only:: not latex
-
-    .. image:: images/modal-classes.webp
-       :width: 584px
-       :alt: A dialog titled Export rows with a "3 selected" badge in its header, a list of three plant names in the body, and a footer containing the text "CSV, UTF-8", a Cancel button and a blue Export button
-
-The following classes can be used by your modal:
-
-``datasette-modal``
-    Added automatically to the native ``<dialog>`` when the wrapper is connected to the page. Provides the dialog's sizing, background, rounded corners, shadow, backdrop and animations.
-
-``modal-header``
-    Adds padding, a bottom border and a horizontal layout for the title and optional metadata.
-
-``modal-title``
-    Sets the title's font size, weight and color. Use ``aria-labelledby`` to associate the title with the dialog.
-
-``modal-meta``
-    Styles optional metadata, such as a selected-item count, as small monospace text with a rounded background.
-
-``modal-body``
-    Adds padding and makes overflowing content scroll while the header and footer remain visible. Sets ``min-height: 0``, ``overflow: auto`` and ``padding: 16px 24px 24px``.
-
-``modal-footer``
-    Adds padding, a top border and a background to the action area. Arranges its contents horizontally, with buttons aligned to the right.
-
-``footer-info``
-    Styles supporting text in the footer and lets it fill the space before the action buttons.
-
-``modal-btn``
-    Provides base button styling, including padding, rounded corners, font and disabled appearance. Use it together with ``modal-btn-primary`` or ``modal-btn-ghost``.
-
-``modal-btn-primary``
-    Gives a button an accent-colored background and white text, suitable for a primary action such as Save.
-
-``modal-btn-ghost``
-    Gives a button a transparent background, muted text and a border, suitable for a secondary action such as Close or Cancel.
-
-These button classes are also used by Datasette's built-in dialogs.
-
-You can customize layout and sizing without adding extra classes. For example, this CSS uses the dialog's existing ID to widen it while keeping it inside the viewport:
-
-.. code-block:: css
-
-    dialog#my-plugin-dialog {
-        width: min(720px, calc(100vw - 32px));
-    }
-
 .. _javascript_datasette_manager_selectors:
 
 Selectors

docs/json_api.rst

@@ -48,7 +48,6 @@ Some JSON endpoints are **exempt** from this promise:
   debug playground.
 - Debug and support endpoints are documented so you can use them, but their
   JSON shapes are not frozen: :ref:`/-/threads <JsonDataView_threads>`,
-  :ref:`/-/tasks <JsonDataView_tasks>`,
   :ref:`/-/actions <JsonDataView_actions>`,
   the :ref:`permission debug endpoints <PermissionsDebugView>`
   (``/-/allowed``, ``/-/rules``, ``/-/check``) and the
@@ -1327,23 +1326,6 @@ The following extras are available for arbitrary SQL query responses and stored,
 
 .. [[[end]]]
 
-.. _TableCountView:
-
-Counting all matching rows
---------------------------
-
-``POST /<database>/<table>/-/count`` returns an exact count of the rows matching the table's query string filters::
-
-    POST /fixtures/facetable/-/count?state=CA
-
-    {"ok": true, "count": 10}
-
-The endpoint supports the same column, search and plugin filters as the table page. Pagination and display options such as ``_next``, ``_size`` and ``_sort`` do not affect the count.
-
-This requires ``view-table`` permission. ``execute-sql`` permission is only needed if using ``_where`` filters.
-
-Unlike the ``count`` extra, this count is not capped by the row count limit. The usual SQL time limit still applies; a timed-out count returns a 400 JSON error.
-
 .. _TableAutocompleteView:
 
 Table autocomplete
@@ -1679,8 +1661,6 @@ The request body is always parsed as JSON, regardless of the request's ``Content
 
 The row-based write APIs can write :ref:`binary values in JSON <binary_json_format>` using Datasette's Base64 representation for BLOB data.
 
-Structured inserts, upserts, updates and deletes only support ordinary SQLite tables. Virtual tables and their internal shadow tables are rejected, including when adding rows to an existing table through the create-table API. Writes to ordinary content tables can still update full-text search indexes through configured triggers.
-
 .. _ExecuteWriteView:
 
 Executing write SQL

docs/plugin_hooks.rst

@@ -217,7 +217,7 @@ Extra template variables that should be made available in the rendered template
 ``datasette`` - :ref:`internals_datasette`
     You can use this to access plugin configuration options via ``datasette.plugin_config(your_plugin_name)``
 
-This hook supports the following return values:
+This hook can return one of three different types:
 
 Dictionary
     If you return a dictionary its keys and values will be merged into the template context.
@@ -228,9 +228,6 @@ Function that returns a dictionary
 Function that returns an awaitable function that returns a dictionary
     You can also return a function which returns an awaitable function which returns a dictionary.
 
-``None``
-    The hook itself, or a function or awaitable it returns, can return ``None`` when no extra variables are needed. Variables returned by other plugins are still included.
-
 Datasette runs Jinja2 in `async mode <https://jinja.palletsprojects.com/en/2.10.x/api/#async-support>`__, which means you can add awaitable functions to the template scope and they will be automatically awaited when they are rendered by the template.
 
 .. warning::
@@ -257,6 +254,8 @@ This example returns an awaitable function which adds a list of ``hidden_table_n
                 return {
                     "hidden_table_names": await db.hidden_table_names()
                 }
+            else:
+                return {}
 
         return hidden_table_names
 
@@ -496,10 +495,10 @@ Lets you customize the display of values within table cells in the HTML table vi
     The name of the column being rendered
 
 ``table`` - string or None
-    The name of the table or view - or ``None`` if this is a custom SQL query
+    The name of the table - or ``None`` if this is a custom SQL query
 
 ``pks`` - list of strings
-    The primary key column names for the table being rendered. For tables without an explicitly defined primary key, this will be ``["rowid"]``. For custom SQL queries and views, this will be an empty list ``[]``.
+    The primary key column names for the table being rendered. For tables without an explicitly defined primary key, this will be ``["rowid"]``. For custom SQL queries and views (where ``table`` is ``None``), this will be an empty list ``[]``.
 
 ``database`` - string
     The name of the database
@@ -1108,7 +1107,7 @@ Return an `ASGI <https://asgi.readthedocs.io/>`__ middleware wrapper function th
 
 This is a very powerful hook. You can use it to manipulate the entire Datasette response, or even to configure new URL routes that will be handled by your own custom code.
 
-You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette <https://starlette.dev/middleware/>`__.
+You can write your ASGI code directly against the low-level specification, or you can use the middleware utilities provided by an ASGI framework such as `Starlette <https://www.starlette.io/middleware/>`__.
 
 This example plugin adds a ``x-databases`` HTTP header listing the currently attached databases:
 
@@ -1158,7 +1157,7 @@ Examples: `datasette-cors <https://datasette.io/plugins/datasette-cors>`__, `dat
 startup(datasette)
 ------------------
 
-This hook fires when the Datasette application server first starts up. It runs on the same event loop that goes on to serve requests, so it is safe to create loop-bound primitives and register background work here — see :ref:`datasette_lifecycle` for the full guarantee and the three ways startup can be triggered.
+This hook fires when the Datasette application server first starts up.
 
 Here is an example that validates required plugin configuration. The server will fail to start and show an error if the validation check fails:
 
@@ -1196,7 +1195,6 @@ Potential use-cases:
 * Create database tables that a plugin needs on startup
 * Validate the configuration for a plugin on startup, and raise an error if it is invalid
 * Raise a ``datasette.utils.StartupError("message")`` exception to prevent Datasette from starting and display that message to the user.
-* Register supervised long-lived background work using :ref:`datasette_add_background_task`, which core launches once every plugin's ``startup()`` hook has finished.
 
 .. note::
 
@@ -1213,31 +1211,6 @@ Potential use-cases:
 
 Examples: `datasette-saved-queries <https://datasette.io/plugins/datasette-saved-queries>`__, `datasette-init <https://datasette.io/plugins/datasette-init>`__
 
-.. _plugin_hook_shutdown:
-
-shutdown(datasette)
--------------------
-
-This hook fires once, when the Datasette application server is shutting down gracefully - triggered by the ASGI ``lifespan.shutdown`` event, which includes pressing Ctrl-C or sending ``SIGTERM`` to a ``datasette serve`` process. It is not called on a hard kill (``SIGKILL``), since there is no opportunity to run any code in that case.
-
-Like ``startup()``, this can be a regular function or it can return an async function to be awaited.
-
-It runs before Datasette cancels any background tasks it is supervising (see :ref:`datasette_add_background_task`) and before it closes its database connections, so you can use it to tell your plugin's own background work to stop gracefully while a database connection is still available to write out any final state. See :ref:`datasette_lifecycle` for exactly where this fits into the full startup-to-shutdown sequence:
-
-.. code-block:: python
-
-    @hookimpl
-    def shutdown(datasette):
-        async def inner():
-            db = datasette.get_database()
-            await db.execute_write(
-                "insert into shutdown_log (at) values (datetime('now'))"
-            )
-
-        return inner
-
-If your ``shutdown()`` hook raises an exception it will be logged but not re-raised, so one plugin's broken shutdown code cannot prevent other plugins - or Datasette itself - from finishing their own teardown.
-
 .. _plugin_hook_actor_from_request:
 
 actor_from_request(datasette, request)

docs/plugin_telemetry.rst

@@ -1,270 +0,0 @@
-.. _plugin_telemetry:
-
-Telemetry for plugin authors
-============================
-
-Datasette core emits OpenTelemetry spans and metrics for the work it does itself - see :ref:`internals_telemetry` for what those are and how an operator turns them on. This page is about the other half: instrumenting the work **your plugin** does, so that a plugin's queries, background jobs and custom operations show up in the same traces and the same metrics pipeline, using the same conventions.
-
-.. _plugin_telemetry_scope:
-
-Use your own instrumentation scope
-----------------------------------
-
-Create a tracer and meter using your plugin's own instrumentation scope:
-
-.. code-block:: python
-
-    from opentelemetry import metrics, trace
-
-    from my_plugin import __version__
-
-    tracer = trace.get_tracer("my_plugin", __version__)
-    meter = metrics.get_meter("my_plugin", __version__)
-
-Use these naming rules:
-
-- **Scope**: use your plugin's import package name, such as ``my_plugin``. This lets users filter telemetry by plugin.
-- **Signal prefix**: prefix spans, metrics and custom attributes with your package name (``my_plugin.*``) or a product name (``paper.*``). The ``datasette.*`` prefix is reserved for core.
-
-Reuse shared attribute names where they describe the same thing: ``db.namespace`` for a database name, or ``error.type`` for an exception class.
-
-If you pass ``schema_url=`` when creating a tracer or meter, choose the semantic-convention version that matches your attributes. Datasette's version is available as ``datasette.telemetry.SCHEMA_URL``. Omit ``schema_url`` if you are unsure which version applies.
-
-.. _plugin_telemetry_registry:
-
-Declare a registry
-------------------
-
-Use ``Attribute``, ``SpanName`` and ``MetricName`` from ``datasette.telemetry_registry`` to describe your plugin's telemetry. Registry entries are strings and can be passed directly to OpenTelemetry:
-
-.. code-block:: python
-
-    from datasette.telemetry_registry import (
-        Attribute,
-        MetricName,
-        SpanName,
-    )
-
-    OUTCOME = Attribute(
-        "my_plugin.outcome",
-        "How the job ended.",
-        values={"ok", "error", "skipped"},
-    )
-    JOB_NAME = Attribute(
-        "my_plugin.job", "The registered job name."
-    )
-
-    JOB_RUN = SpanName(
-        "my_plugin.job.run",
-        "One execution of a scheduled job.",
-        (OUTCOME, JOB_NAME),
-    )
-
-    # A span family with a variable suffix - emitted as "my_plugin.chat gpt-5"
-    CHAT = SpanName(
-        "my_plugin.chat ",
-        "One model call, named ``my_plugin.chat {model}``.",
-        prefix=True,
-    )
-
-    SPANS = (JOB_RUN, CHAT)
-
-    JOB_DURATION = MetricName(
-        "my_plugin.job.duration",
-        "Histogram",
-        "s",
-        "How long each job took.",
-        (JOB_NAME, OUTCOME),
-        buckets=(0.01, 0.1, 1, 10, 60, 600, 3600),
-    )
-
-    METRICS = (JOB_DURATION,)
-
-The example uses these optional arguments:
-
-``values`` - iterable
-    Allowed values for an ``Attribute``. The :ref:`conformance helpers <plugin_telemetry_testing>` check that emitted values belong to this set. Omit it to allow any value.
-
-``prefix`` - boolean
-    For ``SpanName``, match emitted names by prefix. Defaults to ``False``. Exact names take precedence over prefix matches. Avoid overlapping prefixes: the first matching entry in the registry wins.
-
-``buckets`` - iterable
-    Histogram boundaries for a ``MetricName``, expressed in the metric's unit. Pass these to ``meter.create_histogram()`` using ``explicit_bucket_boundaries_advisory=JOB_DURATION.buckets``. Choose boundaries suitable for the operations you measure. For SQLite timings, ``datasette.telemetry_registry.DURATION_BUCKETS`` provides boundaries from 0.0001 to 10 seconds.
-
-.. _plugin_telemetry_privacy:
-
-Privacy and cardinality rules
------------------------------
-
-Core does not explicitly attach bound SQL parameter values, actor identifiers, cookies, authorization headers, client IP addresses or URL query strings as attributes. It does record SQL text, URL paths, host names, User-Agent headers and exception details, which may contain sensitive information. See :ref:`internals_telemetry_privacy`.
-
-- Prefer closed enums, booleans, counts and durations for attribute values. Avoid recording personal information, tokens or other secrets.
-- If you record SQL, use ``datasette.telemetry.sql_attribute()`` on spans only. It truncates SQL text but does not redact literal values. Do not add bound parameter values.
-- Keep metric dimensions bounded. For user input or other unbounded values, record a count, a byte size, a truncation flag or an enum outcome instead.
-
-Use ``assert_no_forbidden_values()`` in :ref:`plugin_telemetry_testing` to check for specific sensitive values in captured telemetry. This helper does not automatically identify all sensitive information.
-
-.. _plugin_telemetry_callbacks:
-
-Your database work is already traced
-------------------------------------
-
-Every call your plugin makes through :ref:`db.execute() <database_execute>`, :ref:`db.execute_fn() <database_execute_fn>`, :ref:`db.execute_write() <database_execute_write>` and :ref:`db.execute_write_fn() <database_execute_write_fn>` already emits core's ``db.query`` spans and is counted in the ``db.client.operation.duration`` histogram. Two consequences:
-
-- **Pass named callables**, not lambdas: the span for a callback-style call is identified by ``datasette.callback``, the callable's qualified name, and a lambda reports ``<lambda>``.
-- If you also wrap those calls in your own span or histogram, you are creating a *second* series in *your* scope - that is fine and sometimes right (yours can carry plugin-level attributes core cannot know), but it is a deliberate two-series design, not a substitute for core's.
-
-.. _plugin_telemetry_request_span:
-
-Enriching the request span
---------------------------
-
-Inside a view or ASGI middleware, ``datasette.telemetry.request_span(scope)`` returns the recording ``SERVER`` span for the current request, or ``None`` when nothing is recording - which is also your signal to skip any work done only to compute attributes:
-
-.. code-block:: python
-
-    from datasette.telemetry import request_span
-
-
-    async def my_view(request):
-        span = request_span(request.scope)
-        if span is not None:
-            span.set_attribute("my_plugin.cache", "hit")
-        ...
-
-.. _plugin_telemetry_background:
-
-Background work: roots with links
----------------------------------
-
-For background work that can outlive a request, create a root span linked to the span that scheduled it. Call ``linked_root_span_kwargs()`` when scheduling the work, then pass the result when starting its span. If there is no valid span context to capture, the new span has no link:
-
-.. code-block:: python
-
-    from datasette.telemetry import linked_root_span_kwargs
-
-    # Capture the current span when scheduling the work:
-    kwargs = linked_root_span_kwargs()
-
-    # Later, wherever the work actually runs:
-    with tracer.start_as_current_span(
-        "my_plugin.job.run", **kwargs
-    ) as span:
-        span.set_attribute(OUTCOME, "ok")
-
-For periodic tasks, create a root span and increment a counter on each iteration, including iterations with no work. Record the result in an outcome attribute. A gauge reporting the time since the last iteration can help monitor tasks with long intervals.
-
-``asyncio.create_task()`` inherits the current trace context. Use ``linked_root_span_kwargs()`` to start background work with its own root span and a link to that context.
-
-Tracers and meters can be created at module scope. In embedded deployments, configure the application's providers before the work you want to record begins.
-
-.. _plugin_telemetry_gauges:
-
-Observable gauges
------------------
-
-Use an observable gauge for current values such as the number of open streams or the length of a queue. The SDK calls its callback when collecting metrics:
-
-- Track live objects using weak references, such as a ``weakref.WeakSet``, and unregister them when they close.
-- Callbacks may run on a different thread from request handlers. Protect shared state and avoid waiting on locks held by request handlers.
-- Read cached state and yield ``Observation`` values. Keep callbacks synchronous and free of I/O. Refresh cached values outside the callback; use a separate gauge to report their age if needed.
-
-Without a provider, gauge callbacks are not invoked.
-
-.. _plugin_telemetry_testing:
-
-Testing your instrumentation
-----------------------------
-
-Use ``datasette.telemetry_testing`` to capture telemetry in your tests and check it against your registry. Add `opentelemetry-sdk <https://github.com/open-telemetry/opentelemetry-python/tree/main/opentelemetry-sdk>`__ to your test dependencies, then import these fixtures in ``conftest.py``:
-
-.. code-block:: python
-
-    from datasette.telemetry_testing import (  # noqa: F401
-        otel_metrics,
-        otel_meter_provider,
-        otel_provider,
-        otel_reset,
-        otel_spans,
-    )
-
-``otel_provider`` and ``otel_meter_provider``
-    Automatically configure in-memory recording for spans and metrics once per test session.
-
-``otel_reset``
-    Automatically clears recorded spans and drains collected metrics after every test.
-
-``otel_spans``
-    Provides an ``InMemorySpanExporter``. Call ``get_finished_spans()`` to retrieve spans recorded during the test.
-
-``otel_metrics``
-    Provides a metrics collector. Call ``collect()`` to capture a snapshot, then use ``point()`` or ``points()`` to inspect it.
-
-Tests requesting ``otel_spans`` or ``otel_metrics`` skip if the SDK is unavailable or another provider has already been installed.
-
-The assertion helpers check the recorded telemetry against your registry:
-
-``assert_spans_conform()``
-    Checks that emitted spans and attributes are registered, and attribute values match any declared ``values=`` enums.
-
-``assert_metrics_conform()``
-    Checks that emitted metrics and attributes are registered, attribute values match any declared enums, and instrument kinds and units match the registry.
-
-``assert_spans_covered()`` and ``assert_metrics_covered()``
-    Check that every registered span or metric and its required attributes appeared during the test. Attributes marked ``optional=True`` are excluded from this check; test those separately.
-
-Pass your plugin's instrumentation scope as ``scope_name`` to these helpers, since the fixtures also record Datasette's own telemetry.
-
-Run a workload that exercises your instrumentation, then call ``otel_metrics.collect()`` once before checking the metrics. Counters and histograms report measurements since the previous collection. Keep the Datasette instance open until collection so observable gauges can report its state:
-
-.. code-block:: python
-
-    from datasette.telemetry_testing import (
-        assert_metrics_conform,
-        assert_metrics_covered,
-        assert_package_never_imports_sdk,
-        assert_spans_covered,
-        assert_spans_conform,
-    )
-
-    from my_plugin.telemetry import METRICS, SPANS
-
-
-    def test_api_only_dependency():
-        assert_package_never_imports_sdk("my_plugin")
-
-
-    def test_conformance(otel_spans, otel_metrics):
-        run_a_workload_that_exercises_everything()
-        finished = otel_spans.get_finished_spans()
-        # Everything emitted is registered (and enum values are legal):
-        assert_spans_conform(
-            SPANS, finished, scope_name="my_plugin"
-        )
-        # Everything registered was emitted:
-        assert_spans_covered(
-            SPANS, finished, scope_name="my_plugin"
-        )
-        # Collect once, then check the metrics:
-        otel_metrics.collect()
-        assert_metrics_conform(
-            METRICS, otel_metrics, scope_name="my_plugin"
-        )
-        assert_metrics_covered(
-            METRICS, otel_metrics, scope_name="my_plugin"
-        )
-
-``assert_package_never_imports_sdk()`` checks that importing your plugin does not import the OpenTelemetry SDK. Run this test early in your suite; see the helper's docstring for a macOS threading limitation.
-
-Use ``assert_no_forbidden_values()`` to check for private data in telemetry. Include fake email addresses, tokens or usernames in your test workload, then pass those values, the finished spans and the collected metrics to the helper. It checks span names, attributes, events, status descriptions and metric attributes.
-
-Leave ``scope_name`` unset for privacy checks so they include both your plugin's telemetry and Datasette's own.
-
-.. _plugin_telemetry_caveats:
-
-Known caveats
--------------
-
-- **Streaming responses hold the request span open.** Core's request span ends when the response body finishes, so for an SSE or long-streaming route its duration is the connection lifetime. If you need per-message timing on a stream, emit your own child spans or span events per message, and use gauges for concurrent-stream counts.
-- **A plugin timing core's work double-measures by design.** See :ref:`plugin_telemetry_callbacks` above.
-- ``datasette.client`` requests made from inside a request produce a nested ``SERVER`` span. Those spans carry ``datasette.internal_client: true`` - filter on it to keep kind-based dashboards from double-counting requests.

docs/plugins.rst

@@ -261,15 +261,6 @@ If you run ``datasette plugins --all`` it will include default plugins that ship
                 "permission_resources_sql"
             ]
         },
-        {
-            "name": "datasette.default_permissions.sqlite_statistics",
-            "static": false,
-            "templates": false,
-            "version": null,
-            "hooks": [
-                "permission_resources_sql"
-            ]
-        },
         {
             "name": "datasette.default_permissions.tokens",
             "static": false,

docs/settings.rst

@@ -67,21 +67,10 @@ The following options can be set using ``--setting name value``, or by storing t
 default_allow_sql
 ~~~~~~~~~~~~~~~~~
 
-.. [[[cog
-    from settings_doc import setting_default
-    setting_default(cog, "default_allow_sql")
-.. ]]]
-
-Default: ``on``
-
-.. [[[end]]]
-
 Should users be able to execute arbitrary SQL queries by default?
 
 Setting this to ``off`` causes permission checks for :ref:`actions_execute_sql` to fail by default.
 
-This setting controls the ability to submit arbitrary SQL. It does not disable structured table-browsing features that use SQL generated by Datasette, such as sorting, column filters and :ref:`facets`. Use :ref:`setting_allow_facet` to control whether users can request facets.
-
 ::
 
     datasette mydatabase.db --setting default_allow_sql off
@@ -93,14 +82,6 @@ Another way to achieve this is to add ``"allow_sql": false`` to your ``datasette
 default_page_size
 ~~~~~~~~~~~~~~~~~
 
-.. [[[cog
-    setting_default(cog, "default_page_size")
-.. ]]]
-
-Default: ``100``
-
-.. [[[end]]]
-
 The default number of rows returned by the table page. You can over-ride this on a per-page basis using the ``?_size=80`` query string parameter, provided you do not specify a value higher than the ``max_returned_rows`` setting. You can set this default using ``--setting`` like so::
 
     datasette mydatabase.db --setting default_page_size 50
@@ -110,15 +91,7 @@ The default number of rows returned by the table page. You can over-ride this on
 sql_time_limit_ms
 ~~~~~~~~~~~~~~~~~
 
-.. [[[cog
+By default, queries have a time limit of one second. If a query takes longer than this to run Datasette will terminate the query and return an error.
-    setting_default(cog, "sql_time_limit_ms")
-.. ]]]
-
-Default: ``1000``
-
-.. [[[end]]]
-
-Time limit for SQL queries, in milliseconds. If a query takes longer than this to run Datasette will terminate the query and return an error.
 
 If this time limit is too short for you, you can customize it using the ``sql_time_limit_ms`` limit - for example, to increase it to 3.5 seconds::
 
@@ -135,15 +108,7 @@ This would set the time limit to 100ms for that specific query. This feature is
 max_returned_rows
 ~~~~~~~~~~~~~~~~~
 
-.. [[[cog
+Datasette returns a maximum of 1,000 rows of data at a time. If you execute a query that returns more than 1,000 rows, Datasette will return the first 1,000 and include a warning that the result set has been truncated. You can use OFFSET/LIMIT or other methods in your SQL to implement pagination if you need to return more than 1,000 rows.
-    setting_default(cog, "max_returned_rows")
-.. ]]]
-
-Default: ``1000``
-
-.. [[[end]]]
-
-The maximum number of rows Datasette returns at a time. If you execute a query that exceeds this limit, Datasette will truncate the result set and include a warning. You can use OFFSET/LIMIT or other methods in your SQL to implement pagination if you need to return more rows.
 
 You can increase or decrease this limit like so::
 
@@ -154,15 +119,7 @@ You can increase or decrease this limit like so::
 max_insert_rows
 ~~~~~~~~~~~~~~~
 
-.. [[[cog
+Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`. Defaults to 100.
-    setting_default(cog, "max_insert_rows")
-.. ]]]
-
-Default: ``100``
-
-.. [[[end]]]
-
-Maximum rows that can be inserted at a time using the bulk insert API, see :ref:`TableInsertView`.
 
 You can increase or decrease this limit like so::
 
@@ -173,15 +130,7 @@ You can increase or decrease this limit like so::
 max_post_body_bytes
 ~~~~~~~~~~~~~~~~~~~
 
-.. [[[cog
+Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API <json_api_write>`. Requests with larger bodies are rejected with an HTTP 413 error. Defaults to 2,097,152 (2MB).
-    setting_default(cog, "max_post_body_bytes")
-.. ]]]
-
-Default: ``2097152``
-
-.. [[[end]]]
-
-Maximum size in bytes for a POST body that Datasette reads fully into memory, such as JSON submitted to the :ref:`write API <json_api_write>`. Requests with larger bodies are rejected with an HTTP 413 error.
 
 This limit exists to protect against memory exhaustion: unlike file uploads handled by ``request.form()``, which stream to disk, these bodies are held entirely in memory and parsing them as JSON can multiply their memory footprint several times over.
 
@@ -198,15 +147,7 @@ Set it to 0 to disable the limit entirely::
 num_sql_threads
 ~~~~~~~~~~~~~~~
 
-.. [[[cog
+Maximum number of threads in the thread pool Datasette uses to execute SQLite queries. Defaults to 3.
-    setting_default(cog, "num_sql_threads")
-.. ]]]
-
-Default: ``3``
-
-.. [[[end]]]
-
-Maximum number of threads in the thread pool Datasette uses to execute SQLite queries.
 
 ::
 
@@ -219,17 +160,9 @@ Setting this to 0 turns off threaded SQL queries entirely - useful for environme
 allow_facet
 ~~~~~~~~~~~
 
-.. [[[cog
-    setting_default(cog, "allow_facet")
-.. ]]]
-
-Default: ``on``
-
-.. [[[end]]]
-
 Allow users to specify columns they would like to facet on using the ``?_facet=COLNAME`` URL parameter to the table view.
 
-If disabled, facets will still be displayed if they have been specifically enabled in ``metadata.json`` configuration for the table.
+This is enabled by default. If disabled, facets will still be displayed if they have been specifically enabled in ``metadata.json`` configuration for the table.
 
 Here's how to disable this feature::
 
@@ -240,15 +173,7 @@ Here's how to disable this feature::
 default_facet_size
 ~~~~~~~~~~~~~~~~~~
 
-.. [[[cog
+The default number of unique rows returned by :ref:`facets` is 30. You can customize it like this::
-    setting_default(cog, "default_facet_size")
-.. ]]]
-
-Default: ``30``
-
-.. [[[end]]]
-
-The default number of unique rows returned by :ref:`facets`. You can customize it like this::
 
     datasette mydatabase.db --setting default_facet_size 50
 
@@ -257,15 +182,7 @@ The default number of unique rows returned by :ref:`facets`. You can customize i
 facet_time_limit_ms
 ~~~~~~~~~~~~~~~~~~~
 
-.. [[[cog
+This is the time limit Datasette allows for calculating a facet, which defaults to 200ms::
-    setting_default(cog, "facet_time_limit_ms")
-.. ]]]
-
-Default: ``200``
-
-.. [[[end]]]
-
-The time limit in milliseconds Datasette allows for calculating a facet. You can customize it like this::
 
     datasette mydatabase.db --setting facet_time_limit_ms 1000
 
@@ -274,15 +191,7 @@ The time limit in milliseconds Datasette allows for calculating a facet. You can
 facet_suggest_time_limit_ms
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-.. [[[cog
+When Datasette calculates suggested facets it needs to run a SQL query for every column in your table. The default for this time limit is 50ms to account for the fact that it needs to run once for every column. If the time limit is exceeded the column will not be suggested as a facet.
-    setting_default(cog, "facet_suggest_time_limit_ms")
-.. ]]]
-
-Default: ``50``
-
-.. [[[end]]]
-
-When Datasette calculates suggested facets it needs to run a SQL query for every column in your table. This time limit, in milliseconds, applies separately to each query. If the time limit is exceeded the column will not be suggested as a facet.
 
 You can increase this time limit like so::
 
@@ -293,15 +202,7 @@ You can increase this time limit like so::
 suggest_facets
 ~~~~~~~~~~~~~~
 
-.. [[[cog
+Should Datasette calculate suggested facets? On by default, turn this off like so::
-    setting_default(cog, "suggest_facets")
-.. ]]]
-
-Default: ``on``
-
-.. [[[end]]]
-
-Should Datasette calculate suggested facets? Turn this off like so::
 
     datasette mydatabase.db --setting suggest_facets off
 
@@ -310,15 +211,7 @@ Should Datasette calculate suggested facets? Turn this off like so::
 allow_download
 ~~~~~~~~~~~~~~
 
-.. [[[cog
+Should users be able to download the original SQLite database using a link on the database index page? This is turned on by default. However, databases can only be downloaded if they are served in immutable mode and not in-memory. If downloading is unavailable for either of these reasons, the download link is hidden even if ``allow_download`` is on. To disable database downloads, use the following::
-    setting_default(cog, "allow_download")
-.. ]]]
-
-Default: ``on``
-
-.. [[[end]]]
-
-Should users be able to download the original SQLite database using a link on the database index page? Databases can only be downloaded if they are served in immutable mode and not in-memory. If downloading is unavailable for either of these reasons, the download link is hidden even if ``allow_download`` is on. To disable database downloads, use the following::
 
     datasette mydatabase.db --setting allow_download off
 
@@ -327,17 +220,9 @@ Should users be able to download the original SQLite database using a link on th
 allow_signed_tokens
 ~~~~~~~~~~~~~~~~~~~
 
-.. [[[cog
-    setting_default(cog, "allow_signed_tokens")
-.. ]]]
-
-Default: ``on``
-
-.. [[[end]]]
-
 Should users be able to create signed API tokens to access Datasette?
 
-Use the following to turn it off::
+This is turned on by default. Use the following to turn it off::
 
     datasette mydatabase.db --setting allow_signed_tokens off
 
@@ -348,17 +233,9 @@ Turning this setting off will disable the ``/-/create-token`` page, :ref:`descri
 max_signed_tokens_ttl
 ~~~~~~~~~~~~~~~~~~~~~
 
-.. [[[cog
-    setting_default(cog, "max_signed_tokens_ttl")
-.. ]]]
-
-Default: ``0``
-
-.. [[[end]]]
-
 Maximum allowed expiry time for signed API tokens created by users.
 
-A value of ``0`` means no limit - tokens can be created that will never expire.
+Defaults to ``0`` which means no limit - tokens can be created that will never expire.
 
 Set this to a value in seconds to limit the maximum expiry time. For example, to set that limit to 24 hours you would use::
 
@@ -371,36 +248,18 @@ This setting is enforced when incoming tokens are processed.
 default_cache_ttl
 ~~~~~~~~~~~~~~~~~
 
-.. [[[cog
+Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-age=X``. Can be over-ridden on a per-request basis using the ``?_ttl=`` query string parameter. Set this to ``0`` to disable HTTP caching entirely. Defaults to 5 seconds.
-    setting_default(cog, "default_cache_ttl")
-.. ]]]
-
-Default: ``5``
-
-.. [[[end]]]
-
-Default HTTP caching max-age header in seconds, used for ``Cache-Control: max-age=X``. Can be over-ridden on a per-request basis using the ``?_ttl=`` query string parameter. Set this to ``0`` to disable HTTP caching entirely.
 
 ::
 
     datasette mydatabase.db --setting default_cache_ttl 60
 
-Dynamic responses for authenticated actors, requests with cookies or an ``Authorization`` header, and responses that set cookies use ``Cache-Control: private, no-store``. This takes precedence over ``default_cache_ttl`` and ``?_ttl=``, even when cache headers are otherwise disabled. Anonymous dynamic responses vary by ``Cookie`` and ``Authorization``. Static assets retain their own cache policy.
-
 .. _setting_cache_size_kb:
 
 cache_size_kb
 ~~~~~~~~~~~~~
 
-.. [[[cog
+Sets the amount of memory SQLite uses for its `per-connection cache <https://www.sqlite.org/pragma.html#pragma_cache_size>`_, in KB.
-    setting_default(cog, "cache_size_kb")
-.. ]]]
-
-Default: ``0``
-
-.. [[[end]]]
-
-Sets the amount of memory SQLite uses for its `per-connection cache <https://www.sqlite.org/pragma.html#pragma_cache_size>`_, in KB. Set this to ``0`` to use SQLite's default cache size.
 
 ::
 
@@ -411,17 +270,9 @@ Sets the amount of memory SQLite uses for its `per-connection cache <https://www
 allow_csv_stream
 ~~~~~~~~~~~~~~~~
 
-.. [[[cog
-    setting_default(cog, "allow_csv_stream")
-.. ]]]
-
-Default: ``on``
-
-.. [[[end]]]
-
 Enables :ref:`the CSV export feature <csv_export>` where an entire table
 (potentially hundreds of thousands of rows) can be exported as a single CSV
-file. You can turn it off like this:
+file. This is turned on by default - you can turn it off like this:
 
 ::
 
@@ -432,16 +283,8 @@ file. You can turn it off like this:
 max_csv_mb
 ~~~~~~~~~~
 
-.. [[[cog
+The maximum size of CSV that can be exported, in megabytes. Defaults to 100MB.
-    setting_default(cog, "max_csv_mb")
+You can disable the limit entirely by settings this to 0:
-.. ]]]
-
-Default: ``100``
-
-.. [[[end]]]
-
-The maximum size of CSV that can be exported, in megabytes.
-You can disable the limit entirely by setting this to 0:
 
 ::
 
@@ -452,14 +295,6 @@ You can disable the limit entirely by setting this to 0:
 truncate_cells_html
 ~~~~~~~~~~~~~~~~~~~
 
-.. [[[cog
-    setting_default(cog, "truncate_cells_html")
-.. ]]]
-
-Default: ``2048``
-
-.. [[[end]]]
-
 In the HTML table view, truncate any strings that are longer than this value.
 The full value will still be available in CSV, JSON and on the individual row
 HTML page. Set this to 0 to disable truncation.
@@ -473,14 +308,6 @@ HTML page. Set this to 0 to disable truncation.
 force_https_urls
 ~~~~~~~~~~~~~~~~
 
-.. [[[cog
-    setting_default(cog, "force_https_urls")
-.. ]]]
-
-Default: ``off``
-
-.. [[[end]]]
-
 Forces self-referential URLs in the JSON output to always use the ``https://``
 protocol. This is useful for cases where the application itself is hosted using
 HTTP but is served to the outside world via a proxy that enables HTTPS.
@@ -494,14 +321,6 @@ HTTP but is served to the outside world via a proxy that enables HTTPS.
 template_debug
 ~~~~~~~~~~~~~~
 
-.. [[[cog
-    setting_default(cog, "template_debug")
-.. ]]]
-
-Default: ``off``
-
-.. [[[end]]]
-
 This setting enables template context debug mode, which is useful to help understand what variables are available to custom templates when you are writing them.
 
 Enable it like this::
@@ -521,14 +340,6 @@ Some examples:
 trace_debug
 ~~~~~~~~~~~
 
-.. [[[cog
-    setting_default(cog, "trace_debug")
-.. ]]]
-
-Default: ``off``
-
-.. [[[end]]]
-
 This setting enables appending ``?_trace=1`` to any page in order to see the SQL queries and other trace information that was used to generate that page.
 
 Enable it like this::
@@ -547,14 +358,6 @@ See :ref:`internals_tracer` for details on how to hook into this mechanism as a
 base_url
 ~~~~~~~~
 
-.. [[[cog
-    setting_default(cog, "base_url")
-.. ]]]
-
-Default: ``/``
-
-.. [[[end]]]
-
 If you are running Datasette behind a proxy, it may be useful to change the root path used for the Datasette instance.
 
 For example, if you are sending traffic from ``https://www.example.com/tools/datasette/`` through to a proxied Datasette instance you may wish Datasette to use ``/tools/datasette/`` as its root URL.

docs/settings_doc.py

@@ -1,10 +0,0 @@
-"""Cog helper for documenting setting defaults from Datasette's registry."""
-
-
-def setting_default(cog, name):
-    from datasette.app import DEFAULT_SETTINGS
-
-    default = DEFAULT_SETTINGS[name]
-    if isinstance(default, bool):
-        default = "on" if default else "off"
-    cog.out(f"\nDefault: ``{default}``\n\n")

docs/shots.yml

@@ -1,38 +0,0 @@
-# Screenshots used by the documentation, taken using shot-scraper:
-# https://shot-scraper.datasette.io/en/stable/multi.html
-#
-# Run "just shots" from the repository root to create any that are
-# missing. Existing images are skipped, so delete an image to recreate it.
-#
-# Paths are relative to this docs/ directory.
-
-# Serves the JavaScript in docs/shots/ and loads it on every page.
-# List form means the datasette process is stopped directly when done.
-- server:
-  - datasette
-  - --memory
-  - --port
-  - 8755
-  - --static
-  - shots:shots
-  - -s
-  - extra_js_urls
-  - '["/shots/modal-example.js", "/shots/modal-classes.js"]'
-
-# javascript_plugins.rst - Reusable modal dialogs
-- output: images/modal-example.webp
-  url: http://localhost:8755/
-  javascript: |
-    document.querySelector('[aria-controls="my-plugin-dialog"]').click();
-  selector: "#my-plugin-dialog"
-  padding: 32
-  quality: 70
-
-- output: images/modal-classes.webp
-  url: http://localhost:8755/
-  javascript: |
-    document.querySelector('[aria-controls="export-dialog"]').click();
-    document.activeElement.blur();
-  selector: "#export-dialog"
-  padding: 32
-  quality: 70

docs/shots/modal-classes.js

@@ -1,41 +0,0 @@
-// Demonstrates every shared modal CSS class, for images/modal-classes.webp
-document.addEventListener("datasette_init", () => {
-    const openButton = document.createElement("button");
-    openButton.type = "button";
-    openButton.textContent = "Open export dialog";
-    openButton.setAttribute("aria-haspopup", "dialog");
-    openButton.setAttribute("aria-controls", "export-dialog");
-
-    const modal = DatasetteModal.create();
-    const dialog = modal.dialog;
-    dialog.id = "export-dialog";
-    dialog.setAttribute("aria-labelledby", "export-dialog-title");
-    dialog.innerHTML = `
-      <div class="modal-header">
-        <h2 class="modal-title" id="export-dialog-title">Export rows</h2>
-        <span class="modal-meta">3 selected</span>
-      </div>
-      <div class="modal-body">
-        <p>Export these rows from the <strong>plants</strong> table as CSV:</p>
-        <ul>
-          <li>Monstera deliciosa</li>
-          <li>Ficus lyrata</li>
-          <li>Pilea peperomioides</li>
-        </ul>
-      </div>
-      <div class="modal-footer">
-        <span class="footer-info">CSV, UTF-8</span>
-        <button type="button" class="modal-btn modal-btn-ghost">Cancel</button>
-        <button type="button" class="modal-btn modal-btn-primary">Export</button>
-      </div>`;
-
-    const [cancelButton, exportButton] = dialog.querySelectorAll(".modal-footer button");
-    cancelButton.addEventListener("click", () => modal.requestClose("cancel"));
-    exportButton.addEventListener("click", () => modal.close());
-    openButton.addEventListener("click", () => {
-        modal.show({ returnFocusTo: openButton, initialFocus: exportButton });
-    });
-
-    document.body.append(modal);
-    document.querySelector("section.content").append(openButton);
-});

docs/shots/modal-example.js

@@ -1,38 +0,0 @@
-document.addEventListener("datasette_init", () => {
-    const openButton = document.createElement("button");
-    openButton.type = "button";
-    openButton.textContent = "Open example dialog";
-    // Indicate that this button opens a dialog:
-    openButton.setAttribute("aria-haspopup", "dialog");
-    // Identify which dialog it controls:
-    openButton.setAttribute("aria-controls", "my-plugin-dialog");
-
-    const modal = DatasetteModal.create();
-    const dialog = modal.dialog;
-    dialog.id = "my-plugin-dialog";
-    // Tell screenreaders the dialog is labelled by #my-plugin-dialog-title
-    dialog.setAttribute("aria-labelledby", "my-plugin-dialog-title");
-    dialog.innerHTML = `
-      <div class="modal-header">
-        <h2 class="modal-title" id="my-plugin-dialog-title">
-          Example dialog
-        </h2>
-      </div>
-      <div class="modal-body">
-        This dialog uses Datasette's shared styles and keyboard behavior.
-      </div>
-      <div class="modal-footer">
-        <button type="button" class="modal-btn modal-btn-ghost">Close</button>
-      </div>`;
-
-    const closeButton = dialog.querySelector("button");
-    closeButton.addEventListener("click", () => {
-        modal.requestClose("cancel");
-    });
-    openButton.addEventListener("click", () => {
-        modal.show({ returnFocusTo: openButton, initialFocus: closeButton });
-    });
-
-    document.body.append(modal);
-    document.querySelector("section.content").append(openButton);
-});

docs/telemetry_doc.py

@@ -1,47 +0,0 @@
-"""
-Cog helpers that render the span and metric reference in ``internals.rst``
-from ``datasette/telemetry_registry.py``.
-"""
-
-
-def _attribute_lines(cog, attributes):
-    if not attributes:
-        cog.out("    No attributes.\n\n")
-        return
-    cog.out("    Attributes:\n\n")
-    for attribute in attributes:
-        suffix = " *(optional)*" if attribute.optional else ""
-        line = f"    - ``{attribute}``{suffix} - {attribute.description}"
-        if attribute.values is not None:
-            rendered = ", ".join(f"``{value}``" for value in sorted(attribute.values))
-            line += f" One of: {rendered}."
-        cog.out(line + "\n")
-    cog.out("\n")
-
-
-def spans(cog):
-    from opentelemetry.trace import SpanKind
-
-    from datasette.telemetry_registry import SPANS
-
-    cog.out("\n")
-    for span in SPANS:
-        cog.out(f"``{span}``\n")
-        cog.out(f"    {span.description}\n\n")
-        # Only show the kind for spans that are not INTERNAL
-        if span.kind != SpanKind.INTERNAL:
-            cog.out(f"    Kind: ``{span.kind.name}``.\n\n")
-        _attribute_lines(cog, span.attributes)
-
-
-def metrics(cog):
-    from datasette.telemetry_registry import METRICS
-
-    cog.out("\n")
-    for metric in METRICS:
-        cog.out(f"``{metric}``\n")
-        cog.out(f"    {metric.kind}, unit ``{metric.unit}``. {metric.description}\n\n")
-        if metric.buckets:
-            boundaries = ", ".join(f"``{boundary}``" for boundary in metric.buckets)
-            cog.out(f"    Bucket boundaries: {boundaries}.\n\n")
-        _attribute_lines(cog, metric.attributes)

docs/testing_plugins.rst

@@ -25,7 +25,7 @@ If you use the template described in :ref:`writing_plugins_cookiecutter` your pl
         )
 
 
-This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX2 <https://httpx2.pydantic.dev/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
+This test uses the :ref:`internals_datasette_client` object to exercise a test instance of Datasette. ``datasette.client`` is a wrapper around the `HTTPX <https://www.python-httpx.org/>`__ Python library which can imitate HTTP requests using ASGI. This is the recommended way to write tests against a Datasette instance.
 
 This test also uses the `pytest-asyncio <https://pypi.org/project/pytest-asyncio/>`__ package to add support for ``async def`` test functions running under pytest.
 
@@ -57,7 +57,7 @@ Then run the tests using pytest like so::
 Setting up a Datasette test instance
 ------------------------------------
 
-Use :ref:`datasette.client <internals_datasette_client>` to make requests against a test instance. The first request runs startup hooks and launches registered background tasks automatically:
+The above example shows the easiest way to start writing tests against a Datasette instance:
 
 .. code-block:: python
 
@@ -71,24 +71,16 @@ Use :ref:`datasette.client <internals_datasette_client>` to make requests agains
         response = await datasette.client.get("/-/plugins.json")
         assert response.status_code == 200
 
-If your test uses Datasette directly without making a request, call ``await datasette.invoke_startup()`` to initialize the instance and run its startup hooks:
+Creating a ``Datasette()`` instance like this as useful shortcut in tests, but there is one detail you need to be aware of. It's important to ensure that the async method ``.invoke_startup()`` is called on that instance. You can do that like this:
 
 .. code-block:: python
 
     datasette = Datasette(memory=True)
     await datasette.invoke_startup()
 
-This runs the :ref:`plugin_hook_startup` and :ref:`plugin_hook_prepare_jinja2_environment` hooks on the same event loop as your test. It does not launch registered background tasks.
+This method registers any :ref:`plugin_hook_startup` or :ref:`plugin_hook_prepare_jinja2_environment` plugins that might themselves need to make async calls.
 
-To run tasks registered with :ref:`datasette_add_background_task` without making a request, use ``await datasette.start_background_tasks()``. This runs startup if needed and launches every registered task:
+If you are using ``await datasette.client.get()`` and similar methods then you don't need to worry about this - Datasette automatically calls ``invoke_startup()`` the first time it handles a request.
-
-.. code-block:: python
-
-    datasette = Datasette(memory=True)
-    await datasette.start_background_tasks()
-    # Tasks registered by startup() hooks have been launched
-
-See :ref:`datasette_lifecycle` for the full startup and shutdown sequence.
 
 .. _testing_plugins_datasette_fixtures_database:
 
@@ -162,7 +154,7 @@ If you need to opt out of this behavior, add the following to your ``pytest.ini`
 Using datasette.client in tests
 -------------------------------
 
-The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX2 async client <https://httpx2.pydantic.dev/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
+The :ref:`internals_datasette_client` mechanism is designed for use in tests. It provides access to a pre-configured `HTTPX async client <https://www.python-httpx.org/async/>`__ instance that can make GET, POST and other HTTP requests against a Datasette instance from inside a test.
 
 A simple test looks like this:
 
@@ -281,22 +273,22 @@ If you want to create that test database repeatedly for every individual test fu
 
 .. _testing_plugins_pytest_httpx:
 
-Testing outbound HTTP calls with pytest-httpx2
+Testing outbound HTTP calls with pytest-httpx
-----------------------------------------------
+---------------------------------------------
 
 If your plugin makes outbound HTTP calls - for example datasette-auth-github or datasette-import-table - you may need to mock those HTTP requests in your tests.
 
-The `pytest-httpx2 <https://pypi.org/project/pytest-httpx2/>`__ package provides a ``httpx2_mock`` fixture, built on `respx <https://lundberg.github.io/respx/>`__, for mocking outbound calls made using HTTPX2.
+The `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__ package is a useful library for mocking calls. It can be tricky to use with Datasette though since it mocks all HTTPX requests, and Datasette's own testing mechanism uses HTTPX internally.
 
-Datasette's own ``datasette.client`` mechanism uses HTTPX2 internally too, but those requests are passed directly to the ASGI application rather than being sent over the network, so they are not affected by the mock.
+To avoid breaking your tests, you can return ``["localhost"]`` from the ``non_mocked_hosts()`` fixture.
 
-As an example, here's a very simple plugin which executes an HTTP request and returns the resulting content:
+As an example, here's a very simple plugin which executes an HTTP response and returns the resulting content:
 
 .. code-block:: python
 
     from datasette import hookimpl
     from datasette.utils.asgi import Response
-    import httpx2
+    import httpx
 
 
     @hookimpl
@@ -314,18 +306,27 @@ As an example, here's a very simple plugin which executes an HTTP request and re
             </form>""")
         vars = await request.post_vars()
         url = vars["url"]
-        return Response.text(httpx2.get(url).text)
+        return Response.text(httpx.get(url).text)
 
-Here's a test for that plugin that mocks the HTTPX2 outbound request:
+Here's a test for that plugin that mocks the HTTPX outbound request:
 
 .. code-block:: python
 
     from datasette.app import Datasette
+    import pytest
 
 
-    async def test_outbound_http_call(httpx2_mock):
+    @pytest.fixture
-        httpx2_mock.get("https://www.example.com/").respond(
+    def non_mocked_hosts():
-            text="Hello world"
+        # This ensures httpx-mock will not affect Datasette's own
+        # httpx calls made in the tests by datasette.client:
+        return ["localhost"]
+
+
+    async def test_outbound_http_call(httpx_mock):
+        httpx_mock.add_response(
+            url="https://www.example.com/",
+            text="Hello world",
         )
         datasette = Datasette([], memory=True)
         response = await datasette.client.post(
@@ -334,13 +335,11 @@ Here's a test for that plugin that mocks the HTTPX2 outbound request:
         )
         assert response.text == "Hello world"
 
-        outbound_request = httpx2_mock.calls.last.request
+        outbound_request = httpx_mock.get_request()
         assert (
             outbound_request.url == "https://www.example.com/"
         )
 
-If your plugin still makes its outbound calls using the original ``httpx`` library you can continue to mock those using `pytest-httpx <https://pypi.org/project/pytest-httpx/>`__.
-
 .. _testing_plugins_register_in_test:
 
 Registering a plugin for the duration of a test

docs/writing_plugins.rst

@@ -203,40 +203,6 @@ Templates should be bundled for distribution using the same ``package_data`` mec
 
 You can also use wildcards here such as ``templates/*.html``. See `datasette-edit-schema <https://github.com/simonw/datasette-edit-schema>`__ for an example of this pattern.
 
-.. _writing_plugins_custom_templates_breadcrumbs:
-
-Adding breadcrumbs
-~~~~~~~~~~~~~~~~~~
-
-Plugin templates that extend ``base.html`` can use the ``crumbs.nav()`` macro to display breadcrumb links back to the Datasette homepage, and optionally to a database and a table. Override the ``crumbs`` block to specify which links to include:
-
-.. code-block:: html+jinja
-
-    {% extends "base.html" %}
-
-    {% block title %}Manage {{ table }}{% endblock %}
-
-    {% block crumbs %}
-    <!-- For home / database / table -->
-    {{ crumbs.nav(request=request, database=database, table=table) }}
-    <!-- For home / database -->
-    {{ crumbs.nav(request=request, database=database) }}
-    {% endblock %}
-
-    {% block content %}
-    <h1>Manage {{ table }}</h1>
-    {% endblock %}
-
-The macro accepts these arguments:
-
-* ``request``: the current request, used to check the actor's permissions.
-* ``database``: an optional database name, as a string
-* ``table``: an optional table name, as a string. If you pass ``table``, you must also pass ``database``.
-
-For a database-level plugin page, use ``{{ crumbs.nav(request=request, database=database) }}``. For a page with just a homepage link, use ``{{ crumbs.nav(request=request) }}``, which is also the default provided by ``base.html`` if you do not override the block.
-
-The table-level example renders links in the form ``home / database / table``. Each link is only included if the current actor has permission to view that resource.
-
 .. _writing_plugins_configuration:
 
 Writing plugins that accept configuration

pyproject.toml

@@ -28,9 +28,9 @@ dependencies = [
     "click-default-group>=1.2.3",
     "Jinja2>=2.10.3",
     "hupper>=1.9",
-    "httpx2>=2.0",
+    "httpx>=0.20,<1.0",
     "pluggy>=1.0",
-    "uvicorn>=0.29",
+    "uvicorn>=0.11",
     "aiofiles>=0.4",
     "PyYAML>=5.3",
     "mergedeep>=1.1.1",
@@ -40,7 +40,6 @@ dependencies = [
     "setuptools",
     "pip",
     "pydantic>=2",
-    "opentelemetry-api>=1.37",
 ]
 
 [project.urls]
@@ -64,17 +63,16 @@ dev = [
     "pytest-xdist>=2.2.1",
     "pytest-asyncio>=1.2.0",
     "beautifulsoup4>=4.8.1",
-    "black==26.5.1",
+    "black==26.3.1",
     "blacken-docs==1.20.0",
     "pytest-timeout>=1.4.2",
     "trustme>=0.7",
     "cogapp>=3.3.0",
     "multipart-form-data-conformance==0.1a0",
     "ruff>=0.16.0",
-    "opentelemetry-sdk>=1.37",
     # docs
     "Sphinx==7.4.7",
-    "furo==2025.12.19",
+    "furo==2025.9.25",
     "sphinx-autobuild",
     "codespell>=2.2.5",
     "sphinx-copybutton",
@@ -87,9 +85,6 @@ dev = [
 playwright = [
     "pytest-playwright>=0.8.0",
 ]
-shots = [
-    "shot-scraper>=1.12",
-]
 
 [project.optional-dependencies]
 rich = ["rich"]

pytest.ini

@@ -1,5 +1,4 @@
 [pytest]
-addopts = --ignore=ignored
 filterwarnings=
     # https://github.com/pallets/jinja/issues/927
     ignore:Using or importing the ABCs::jinja2

tests/conftest.py

@@ -2,14 +2,13 @@ import importlib.metadata
 import os
 import pathlib
 import re
-import socket
 import subprocess
 import sys
 import tempfile
 import time
 from dataclasses import dataclass
 
-import httpx2
+import httpx
 import pytest
 import pytest_asyncio
 
@@ -33,41 +32,17 @@ UNDOCUMENTED_PERMISSIONS = {
 }
 
 
-def wait_until_responds(url, timeout=5.0, client=httpx2, process=None, **kwargs):
+def wait_until_responds(url, timeout=5.0, client=httpx, **kwargs):
     start = time.time()
     while time.time() - start < timeout:
-        # If the server died there is no point waiting out the timeout - fail
-        # now, with its output, instead of after `timeout` seconds of silence
-        if process is not None and process.poll() is not None:
-            raise AssertionError(
-                "Server exited early with returncode {}\n{}".format(
-                    process.returncode, process.stdout.read().decode("utf-8")
-                )
-            )
         try:
             client.get(url, **kwargs)
             return
-        except httpx2.TransportError:
+        except httpx.ConnectError:
             time.sleep(0.1)
     raise AssertionError(f"Timed out waiting for {url} to respond")
 
 
-def find_free_port():
-    with socket.socket() as sock:
-        sock.bind(("127.0.0.1", 0))
-        return sock.getsockname()[1]
-
-
-from datasette.telemetry_testing import (  # noqa: F401
-    MetricsCollector,
-    otel_meter_provider,
-    otel_metrics,
-    otel_provider,
-    otel_reset,
-    otel_spans,
-)
-
-
 @pytest.fixture
 def bare_ds():
     """
@@ -116,10 +91,7 @@ async def ds_client():
 
     await db.execute_write_fn(prepare)
     await ds.invoke_startup()
-    try:
+    return ds.client
-        yield ds.client
-    finally:
-        ds.close()
 
 
 def pytest_report_header(config):
@@ -181,11 +153,6 @@ def pytest_collection_modifyitems(config, items):
     move_to_front(items, "test_spatialite_error_if_attempt_to_open_spatialite")
     move_to_front(items, "test_package")
     move_to_front(items, "test_package_with_port")
-    # These start subprocesses, which can crash on macOS/CPython 3.13 late in
-    # a test run once the pytest process has started many threads
-    move_to_front(items, "test_datasette_package_never_imports_the_sdk")
-    move_to_front(items, "test_kit_module_itself_never_imports_the_sdk")
-    move_to_front(items, "test_no_provider_takes_the_fast_path")
 
 
 def move_to_front(items, test_name):
@@ -284,24 +251,12 @@ def ds_localhost_http_server():
         # Avoid FileNotFoundError: [Errno 2] No such file or directory:
         cwd=tempfile.gettempdir(),
     )
-    try:
+    wait_until_responds("http://localhost:8041/")
-        wait_until_responds("http://localhost:8041/", process=ds_proc)
+    # Check it started successfully
+    assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8")
     yield ds_proc
-    finally:
+    # Shut it down at the end of the pytest session
-        stop_process(ds_proc)
+    ds_proc.terminate()
-
-
-def stop_process(proc):
-    try:
-        if proc.poll() is None:
-            proc.terminate()
-        try:
-            proc.wait(timeout=5)
-        except subprocess.TimeoutExpired:
-            proc.kill()
-            proc.wait()
-    finally:
-        proc.stdout.close()
 
 
 @pytest.fixture(scope="session")
@@ -322,27 +277,11 @@ def ds_unix_domain_socket_server(tmp_path_factory):
         cwd=tempfile.gettempdir(),
     )
     # Poll until available
-    transport = httpx2.HTTPTransport(uds=uds)
+    transport = httpx.HTTPTransport(uds=uds)
-    client = httpx2.Client(transport=transport)
+    client = httpx.Client(transport=transport)
     try:
-        # Probe with a socket we own: the HTTP transport can leak a socket
-        # when connect() fails before the UDS server has started listening.
-        start = time.monotonic()
-        while True:
-            try:
-                with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as probe:
-                    probe.settimeout(0.1)
-                    probe.connect(uds)
-                break
-            except OSError:
-                if ds_proc.poll() is not None or time.monotonic() - start > 30:
-                    raise
-                time.sleep(0.1)
         wait_until_responds(
-            "http://localhost/_memory.json",
+            "http://localhost/_memory.json", timeout=30.0, client=client
-            timeout=30.0,
-            client=client,
-            process=ds_proc,
         )
         # Check it started successfully
         assert not ds_proc.poll(), ds_proc.stdout.read().decode("utf-8")
@@ -350,72 +289,18 @@ def ds_unix_domain_socket_server(tmp_path_factory):
     finally:
         client.close()
         # Shut it down at the end of the pytest session
-        stop_process(ds_proc)
+        ds_proc.terminate()
+        try:
+            ds_proc.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            ds_proc.kill()
+            ds_proc.wait()
         try:
             os.unlink(uds)
         except FileNotFoundError:
             pass
 
 
-@pytest.fixture
-def serve_with_plugins(tmp_path):
-    """Factory fixture for starting ``datasette serve`` in a subprocess with
-    plugins written to a temporary ``--plugins-dir``.
-
-    For tests that need the real serve path: event-loop wiring, exit codes,
-    signals. The usual in-process ``pm.register`` plugin pattern can't reach
-    a subprocess, so plugin source is written out as importable files instead.
-
-    Unlike ``ds_localhost_http_server`` this is function-scoped and takes a
-    fresh port each time, because each test needs its own plugins. Call it as::
-
-        proc, port = serve_with_plugins({"my_plugin": PLUGIN_SOURCE})
-
-    ``plugins`` maps module name to Python source. Pass
-    ``wait_for_startup=False`` when the server is expected to fail during
-    startup rather than begin serving. Extra CLI arguments are passed through.
-    Every process started is terminated when the test ends.
-    """
-    processes = []
-
-    def start(plugins, *extra_args, wait_for_startup=True):
-        plugins_dir = tmp_path / "plugins"
-        plugins_dir.mkdir(exist_ok=True)
-        for module_name, source in plugins.items():
-            (plugins_dir / f"{module_name}.py").write_text(source, "utf-8")
-        port = find_free_port()
-        proc = subprocess.Popen(
-            [
-                sys.executable,
-                "-m",
-                "datasette",
-                "--memory",
-                "--plugins-dir",
-                str(plugins_dir),
-                "-h",
-                "127.0.0.1",
-                "-p",
-                str(port),
-                *extra_args,
-            ],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            # Avoid FileNotFoundError: [Errno 2] No such file or directory:
-            cwd=tempfile.gettempdir(),
-        )
-        processes.append(proc)
-        if wait_for_startup:
-            wait_until_responds(
-                f"http://127.0.0.1:{port}/-/versions.json", process=proc
-            )
-        return proc, port
-
-    yield start
-
-    for proc in processes:
-        stop_process(proc)
-
-
 # Import fixtures from fixtures.py to make them available
 from .fixtures import (  # noqa: F401
     TEMP_PLUGIN_SECRET_FILE,

tests/fixtures.py

@@ -169,10 +169,12 @@ def make_app_client(
             template_dir=template_dir,
             crossdb=crossdb,
         )
-        try:
         yield TestClient(ds)
-        finally:
+        # Close as many database connections as possible
-            ds.close()
+        # to try and avoid too many open files error
+        for db in ds.databases.values():
+            if not db.is_memory:
+                db.close()
 
 
 @pytest.fixture(scope="session")
@@ -184,10 +186,9 @@ def app_client():
 @pytest.fixture(scope="session")
 def app_client_no_files():
     ds = Datasette([])
-    try:
     yield TestClient(ds)
-    finally:
+    for db in ds.databases.values():
-        ds.close()
+        db.close()
 
 
 @pytest.fixture(scope="session")

tests/test_api.py

@@ -5,7 +5,6 @@ import pytest
 
 from datasette.app import Datasette
 from datasette.plugins import DEFAULT_PLUGINS
-from datasette.resources import DatabaseResource, TableResource
 from datasette.utils import UNSTABLE_API_MESSAGE, escape_sqlite, tilde_encode
 from datasette.utils.sqlite import sqlite_version
 from datasette.version import __version__
@@ -102,11 +101,14 @@ async def test_database_page(ds_client):
         "tags",
     }
 
-    # The external-content index is visible, but its shadow tables need a
+    # Expected hidden tables
-    # second dependency hop and are excluded by the one-hop permission policy.
     expected_hidden_tables = {
         "no_primary_key",
         "searchable_fts",
+        "searchable_fts_config",
+        "searchable_fts_data",
+        "searchable_fts_docsize",
+        "searchable_fts_idx",
     }
 
     # Verify all expected tables exist
@@ -456,67 +458,6 @@ async def test_row_foreign_key_tables(ds_client):
     ]
 
 
-@pytest.mark.asyncio
-async def test_row_foreign_key_tables_omit_denied_tables(request):
-    actor = {"id": "reader"}
-    ds = Datasette(
-        memory=True,
-        default_deny=True,
-        config={
-            "databases": {
-                "data": {
-                    "tables": {
-                        "parents": {"permissions": {"view-table": True}},
-                        "private_children": {"permissions": {"view-table": False}},
-                    }
-                }
-            }
-        },
-    )
-    request.addfinalizer(ds.close)
-    db = ds.add_memory_database("fk_count_leak", name="data")
-    await db.execute_write("create table parents (id integer primary key, name text)")
-    await db.execute_write("""
-        create table private_children (
-            id integer primary key,
-            parent_id integer references parents(id)
-        )
-    """)
-    await db.execute_write("insert into parents values (1, 'Public parent')")
-    await db.execute_write("""
-        insert into private_children (id, parent_id) values
-            (1, 1),
-            (2, 1),
-            (3, 1)
-    """)
-    await ds.invoke_startup()
-
-    parent = TableResource(database="data", table="parents")
-    private_children = TableResource(database="data", table="private_children")
-    assert await ds.allowed(action="view-table", resource=parent, actor=actor)
-    assert not await ds.allowed(
-        action="view-table", resource=private_children, actor=actor
-    )
-    assert not await ds.allowed(
-        action="execute-sql",
-        resource=DatabaseResource(database="data"),
-        actor=actor,
-    )
-
-    direct_child = await ds.client.get("/data/private_children.json", actor=actor)
-    assert direct_child.status_code == 403
-    parent_response = await ds.client.get(
-        "/data/parents/1.json?_extra=foreign_key_tables", actor=actor
-    )
-    assert parent_response.status_code == 200
-
-    foreign_key_tables = parent_response.json().get("foreign_key_tables", [])
-    assert foreign_key_tables == [], (
-        "denied child table name, foreign-key column, and row count disclosed: "
-        f"{foreign_key_tables}"
-    )
-
-
 @pytest.mark.asyncio
 async def test_row_extras(ds_client):
     response = await ds_client.get(
@@ -953,7 +894,10 @@ async def test_hidden_sqlite_stat1_table():
     await db.execute_write("analyze")
     data = (await ds.client.get("/db.json?_show_hidden=1")).json()
     tables = [(t["name"], t["hidden"]) for t in data["tables"]]
-    assert tables == [("normal", False)]
+    assert tables in (
+        [("normal", False), ("sqlite_stat1", True)],
+        [("normal", False), ("sqlite_stat1", True), ("sqlite_stat4", True)],
+    )
 
 
 @pytest.mark.asyncio

tests/test_api_write.py

@@ -1,7 +1,6 @@
 import time
 
 import pytest
-import sqlite_utils
 
 from datasette.app import Datasette
 from datasette.events import RenameTableEvent
@@ -56,105 +55,6 @@ def _headers(token):
     }
 
 
-@pytest.mark.asyncio
-@pytest.mark.parametrize("operation", ["read", "read_row", "rename"])
-async def test_trailing_lf_table_permissions(tmp_path, operation):
-    # SQLite treats "secret" and "secret\n" as different table names. Permission
-    # checks and SQL execution must agree on which table a request targets.
-    db_path = tmp_path / "data.db"
-    conn = sqlite3.connect(str(db_path))
-    conn.executescript(
-        "create table secret (id integer primary key, value text);"
-        "insert into secret values (1, 'private');"
-    )
-    conn.close()
-    # Allow builder to create and use tables generally, but explicitly deny
-    # access to the existing secret table below. Disable arbitrary SQL access.
-    grants = {
-        action: {"id": "builder"}
-        for action in (
-            "view-database",
-            "create-table",
-            "view-table",
-            "insert-row",
-            "alter-table",
-        )
-    }
-    ds = Datasette(
-        [str(db_path)],
-        default_deny=True,
-        settings={"default_allow_sql": False},
-        config={
-            "permissions": {"view-instance": {"id": "builder"}},
-            "databases": {
-                "data": {
-                    "permissions": grants,
-                    "tables": {
-                        "secret": {
-                            "permissions": {
-                                "view-table": False,
-                                "insert-row": False,
-                                "alter-table": False,
-                            }
-                        }
-                    },
-                }
-            },
-        },
-    )
-    headers = _headers(write_token(ds, actor_id="builder"))
-    try:
-        # Establish that the protected table is inaccessible before creating
-        # a second table whose name differs only by a trailing line feed.
-        response = await ds.client.get("/data/secret.json", headers=headers)
-        assert response.status_code == 403
-        response = await ds.client.get(
-            "/data/-/query.json?sql=select+*+from+secret", headers=headers
-        )
-        assert response.status_code == 403
-        # Distinct values let us detect if an operation targets secret
-        # instead of the newly created secret\n table.
-        response = await ds.client.post(
-            "/data/-/create",
-            json={"table": "secret\n", "row": {"id": 1, "value": "decoy"}, "pk": "id"},
-            headers=headers,
-        )
-        assert response.status_code == 201, response.text
-        # ~0A is Datasette's URL encoding for the line feed in the table name.
-        if operation in ("read", "read_row"):
-            # Both table and row endpoints must return only the permitted row.
-            path = "/1.json" if operation == "read_row" else ".json"
-            response = await ds.client.get(
-                "/data/secret~0A" + path + "?_shape=array", headers=headers
-            )
-            assert response.status_code == 200, response.text
-            assert response.json() == [{"id": 1, "value": "decoy"}]
-        else:
-            # Renaming must move the permitted table, preserving its contents
-            # and removing its old name from the database.
-            response = await ds.client.post(
-                "/data/secret~0A/-/alter",
-                json={"operations": [{"op": "rename_table", "args": {"to": "moved"}}]},
-                headers=headers,
-            )
-            assert response.status_code == 200, response.text
-            db = ds.get_database("data")
-            assert (
-                await db.execute('select value from "moved"')
-            ).single_value() == "decoy"
-            assert "secret\n" not in await db.table_names()
-        # Verify that the protected table and its data are unchanged, and that
-        # the API still denies access to it.
-        db = ds.get_database("data")
-        assert (
-            await db.execute('select value from "secret"')
-        ).single_value() == "private"
-        response = await ds.client.get("/data/secret.json", headers=headers)
-        assert response.status_code == 403
-    finally:
-        ds.close()
-
-
 def _insert_and_fetch_created(conn, table, insert_sql):
     cursor = conn.execute(insert_sql)
     return conn.execute(
@@ -167,82 +67,6 @@ BASE64_WRITE_API_VALUE = {"$base64": True, "encoded": "AAEC/f7/"}
 BASE64_WRITE_API_LITERAL = '{"$base64": true, "encoded": "AAEC/f7/"}'
 
 
-@pytest.mark.asyncio
-@pytest.mark.parametrize("use_fallback", (False, True))
-@pytest.mark.parametrize(
-    "operation", ("insert", "upsert", "update", "delete", "create", "create_uppercase")
-)
-@pytest.mark.parametrize(
-    "module,definition,values,shadow_suffix",
-    (
-        ("fts5", "body", "'original'", "_content"),
-        ("fts4", "body", "'original'", "_content"),
-        ("rtree", "id, minx, maxx", "1, 0, 1", "_rowid"),
-    ),
-)
-@pytest.mark.parametrize("shadow", (False, True))
-async def test_structured_writes_require_ordinary_tables(
-    ds_write,
-    monkeypatch,
-    use_fallback,
-    operation,
-    module,
-    definition,
-    values,
-    shadow_suffix,
-    shadow,
-):
-    if use_fallback:
-        monkeypatch.setattr("datasette.utils.sqlite.supports_table_list", lambda: False)
-    db = ds_write.get_database("data")
-    await db.execute_write(f"create virtual table indexed using {module}({definition})")
-    await db.execute_write(f"insert into indexed values ({values})")
-    table = "indexed" + (shadow_suffix if shadow else "")
-    row = (await db.execute(f"select rowid, * from {escape_sqlite(table)}")).dicts()[0]
-    pks = await db.primary_keys(table)
-    pk_value = row[pks[0] if pks else "rowid"]
-    before = await db.execute_fn(lambda conn: list(conn.iterdump()))
-
-    if operation in ("create", "create_uppercase"):
-        path = "/data/-/create"
-        body = {
-            "table": table.upper() if operation == "create_uppercase" else table,
-            "rows": [row],
-        }
-    elif operation in ("update", "delete"):
-        path = f"/data/{table}/{pk_value}/-/{operation}"
-        body = {"update": row} if operation == "update" else {}
-    else:
-        path = f"/data/{table}/-/{operation}"
-        body = {"rows": [row]}
-    response = await ds_write.client.post(
-        path, json=body, headers=_headers(write_token(ds_write))
-    )
-    assert response.status_code == 400, response.text
-    assert response.json()["errors"] == ["Structured writes require an ordinary table"]
-    assert await db.execute_fn(lambda conn: list(conn.iterdump())) == before
-
-
-@pytest.mark.asyncio
-async def test_structured_writes_to_content_table_maintain_fts(ds_write):
-    db = ds_write.get_database("data")
-    await db.execute_write_fn(
-        lambda conn: sqlite_utils.Database(conn)["docs"].enable_fts(
-            ["title"], create_triggers=True
-        )
-    )
-    response = await ds_write.client.post(
-        "/data/docs/-/insert",
-        json={"row": {"id": 1, "title": "ordinary content"}},
-        headers=_headers(write_token(ds_write)),
-    )
-    assert response.status_code == 201, response.text
-    matches = await db.execute(
-        "select rowid from docs_fts where docs_fts match ?", ["ordinary"]
-    )
-    assert [row[0] for row in matches.rows] == [1]
-
-
 @pytest.mark.asyncio
 async def test_base64_write_api_create_table_infers_blob_and_raw_escapes(ds_write):
     token = write_token(ds_write)
@@ -1471,7 +1295,7 @@ async def test_alter_table_foreign_key_without_fk_column_requires_single_pk(ds_w
 
 @pytest.mark.asyncio
 async def test_foreign_key_suggestions(ds_write):
-    token = write_token(ds_write, permissions=["alter-table", "view-table"])
+    token = write_token(ds_write, permissions=["at"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
     await db.execute_write("insert into owners (id) values (1), (2), (3)")
@@ -1537,7 +1361,7 @@ async def test_foreign_key_suggestions_permission_denied(ds_write):
 
 @pytest.mark.asyncio
 async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch):
-    token = write_token(ds_write, permissions=["alter-table", "view-table"])
+    token = write_token(ds_write, permissions=["at"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
 
@@ -1568,7 +1392,7 @@ async def test_foreign_key_suggestions_fail_open(ds_write, monkeypatch):
 
 @pytest.mark.asyncio
 async def test_foreign_key_targets(ds_write):
-    token = write_token(ds_write, permissions=["create-table", "view-table"])
+    token = write_token(ds_write, permissions=["ct"])
     db = ds_write.get_database("data")
     await db.execute_write("create table owners (id integer primary key)")
     await db.execute_write("create table categories (slug varchar(30) primary key)")
@@ -1901,42 +1725,6 @@ async def test_drop_table(ds_write, scenario):
         assert (await ds_write.client.get("/data/docs")).status_code == 404
 
 
-@pytest.mark.asyncio
-async def test_drop_table_cleans_up_fts(ds_write):
-    db = ds_write.get_database("data")
-
-    def enable_fts(conn):
-        sqlite_utils.Database(conn)["docs"].enable_fts(["title"], create_triggers=True)
-
-    await db.execute_write_fn(enable_fts)
-    assert {
-        row[0]
-        for row in await db.execute(
-            "select name from sqlite_master where type = 'table' and name like 'docs_fts%'"
-        )
-    } == {
-        "docs_fts",
-        "docs_fts_config",
-        "docs_fts_data",
-        "docs_fts_docsize",
-        "docs_fts_idx",
-    }
-
-    response = await ds_write.client.post(
-        "/data/docs/-/drop",
-        json={"confirm": True},
-        headers=_headers(write_token(ds_write)),
-    )
-
-    assert response.json() == {"ok": True}
-    assert [
-        row[0]
-        for row in await db.execute(
-            "select name from sqlite_master where type = 'table' and name like 'docs_fts%'"
-        )
-    ] == []
-
-
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "input,expected_status,expected_response,expected_events",
@@ -2920,119 +2708,3 @@ async def test_create_using_alter_against_existing_table(
         insert_rows_event = ds_write._tracked_events[1]
         assert insert_rows_event.name == "insert-rows"
         assert insert_rows_event.num_rows == 1
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize(
-    ("denied_action", "request_body"),
-    (
-        (
-            "insert-row",
-            {
-                "table": "salaries",
-                "rows": [{"id": 9, "note": "INJ-VIA-CREATE"}],
-            },
-        ),
-        (
-            "update-row",
-            {
-                "table": "salaries",
-                "rows": [{"id": 1, "note": "REPLACED"}],
-                "pk": "id",
-                "replace": True,
-            },
-        ),
-        (
-            "alter-table",
-            {
-                "table": "salaries",
-                "rows": [{"id": 9, "note": "INSERTED", "extra": "NEW"}],
-                "alter": True,
-            },
-        ),
-    ),
-)
-async def test_create_table_existing_table_respects_table_level_denial(
-    denied_action, request_body
-):
-    # GHSA-53fc-rhfg-h7qp issue 2: POST /db/-/create against an existing table
-    # inserts rows into it, so insert-row (and update-row / alter-table) must be
-    # checked against the TableResource, not just the DatabaseResource.
-    ds = Datasette(
-        memory=True,
-        config={
-            "databases": {
-                # id=editor user has each permission at the database level, but
-                # the selected action is explicitly denied on the salaries table
-                "data": {
-                    "permissions": {
-                        "create-table": {"id": "editor"},
-                        "insert-row": {"id": "editor"},
-                        "update-row": {"id": "editor"},
-                        "alter-table": {"id": "editor"},
-                    },
-                    "tables": {
-                        "salaries": {"permissions": {denied_action: False}},
-                    },
-                }
-            }
-        },
-    )
-    db = ds.add_memory_database(
-        f"create_table_existing_table_denied_{denied_action}", name="data"
-    )
-    await db.execute_write("create table salaries (id integer primary key, note text)")
-    await db.execute_write("insert into salaries values (1, 'TOPSECRET-A')")
-    await ds.invoke_startup()
-
-    if denied_action == "insert-row":
-        # Sanity: direct insert into salaries is denied for this actor
-        direct = await ds.client.post(
-            "/data/salaries/-/insert",
-            actor={"id": "editor"},
-            json={"row": {"id": 9, "note": "INJ-DIRECT"}},
-        )
-        assert direct.status_code == 403
-
-    response = await ds.client.post(
-        "/data/-/create",
-        actor={"id": "editor"},
-        json=request_body,
-    )
-    assert response.status_code == 403, response.json()
-    assert response.json()["errors"] == [f"Permission denied: need {denied_action}"]
-    rows = (await db.execute("select id, note from salaries order by id")).rows
-    assert [tuple(r) for r in rows] == [(1, "TOPSECRET-A")]
-    assert await db.table_columns("salaries") == ["id", "note"]
-
-
-@pytest.mark.asyncio
-async def test_create_table_respects_predeclared_table_level_denial():
-    ds = Datasette(
-        memory=True,
-        config={
-            "databases": {
-                "data": {
-                    "permissions": {
-                        "create-table": {"id": "editor"},
-                        "insert-row": {"id": "editor"},
-                    },
-                    "tables": {
-                        "planned_table": {"permissions": {"insert-row": False}},
-                    },
-                }
-            }
-        },
-    )
-    db = ds.add_memory_database("create_table_predeclared_denial", name="data")
-    await ds.invoke_startup()
-
-    response = await ds.client.post(
-        "/data/-/create",
-        actor={"id": "editor"},
-        json={"table": "planned_table", "rows": [{"id": 1}]},
-    )
-
-    assert response.status_code == 403, response.json()
-    assert response.json()["errors"] == ["Permission denied: need insert-row"]
-    assert not await db.table_exists("planned_table")

tests/test_auth.py

@@ -1,5 +1,4 @@
 import time
-from unittest.mock import AsyncMock
 
 import pytest
 from bs4 import BeautifulSoup as Soup
@@ -238,35 +237,6 @@ def test_auth_create_token(
         assert response3.json["actor"]["id"] == "test"
 
 
-@pytest.mark.asyncio
-@pytest.mark.parametrize("method", ["GET", "POST"])
-@pytest.mark.parametrize(
-    "restrictions",
-    [
-        {},
-        {"a": ["vi"]},
-        {"d": {"db": ["vd"]}},
-        {"r": {"db": {"t1": ["vt"]}}},
-    ],
-    ids=["empty", "instance", "database", "table"],
-)
-async def test_auth_create_token_not_allowed_for_restricted_actors(
-    bare_ds, monkeypatch, method, restrictions
-):
-    create_token = AsyncMock()
-    monkeypatch.setattr(bare_ds, "create_token", create_token)
-
-    response = await bare_ds.client.request(
-        method,
-        "/-/create-token",
-        actor={"id": "test", "_r": restrictions},
-    )
-
-    assert response.status_code == 403
-    assert "Restricted actors cannot create API tokens" in response.text
-    create_token.assert_not_called()
-
-
 @pytest.mark.asyncio
 async def test_auth_create_token_not_allowed_for_tokens(ds_client):
     ds_tok = ds_client.ds.sign(
@@ -554,25 +524,3 @@ async def test_root_without_root_enabled_no_special_permissions(ds_client):
         )
         is not True
     ), "Root without root_enabled should not automatically get set-column-type"
-
-
-@pytest.mark.parametrize("expire_after", (1, 300, 3600, 30 * 24 * 60 * 60))
-def test_set_actor_cookie_honours_expire_after(expire_after):
-    # GHSA-53fc-rhfg-h7qp issue 4: expire_after is documented as a number of
-    # seconds, but every value was being replaced with 24 hours.
-    from datasette.app import Datasette
-    from datasette.utils.asgi import Response
-
-    ds = Datasette(memory=True)
-    response = Response.text("")
-    before = int(time.time())
-    ds.set_actor_cookie(response, {"id": "test"}, expire_after=expire_after)
-    after = int(time.time())
-
-    (header,) = response._set_cookie_headers
-    assert header.startswith("ds_actor=")
-    value = header[len("ds_actor=") :].split(";", 1)[0]
-    data = ds.unsign(value, "actor")
-    assert data["a"] == {"id": "test"}
-    expires_at = baseconv.base62.decode(data["e"])
-    assert before + expire_after <= expires_at <= after + expire_after

tests/test_background_tasks.py

@@ -1,381 +0,0 @@
-"""
-Tests for datasette.add_background_task() / start_background_tasks() and the
-BackgroundTask / BackgroundTaskSupervisor machinery in
-datasette/background_tasks.py.
-"""
-
-import asyncio
-import contextlib
-import logging
-
-import httpx2
-import pytest
-
-from datasette import hookimpl
-from datasette.app import Datasette
-from datasette.plugins import pm
-
-
-async def _drive_lifespan_startup(app):
-    """Send a single lifespan.startup message into app's ASGI lifespan loop
-    and return the list of messages sent back, without ever sending
-    lifespan.shutdown. Copied from tests/test_lifespan.py's helper of the
-    same name - mirrors what a real server does: after startup completes
-    it parks waiting for the next event, and we cancel that wait once
-    we've observed the startup response.
-    """
-    messages_sent = []
-    startup_responded = asyncio.Event()
-    delivered = False
-
-    async def receive():
-        nonlocal delivered
-        if not delivered:
-            delivered = True
-            return {"type": "lifespan.startup"}
-        await asyncio.Event().wait()
-
-    async def send(message):
-        messages_sent.append(message)
-        startup_responded.set()
-
-    task = asyncio.create_task(app({"type": "lifespan"}, receive, send))
-    try:
-        await asyncio.wait_for(startup_responded.wait(), timeout=5)
-    finally:
-        task.cancel()
-        with contextlib.suppress(asyncio.CancelledError):
-            await task
-    return messages_sent
-
-
-@pytest.mark.asyncio
-async def test_tasks_registered_in_startup_hook_run_after_lifespan_startup():
-    # Two tasks registered by one plugin's startup hook - order preserved,
-    # both running after lifespan startup completes, and no HTTP request
-    # of any kind is issued anywhere in this test.
-    events = []
-
-    async def task_one(datasette):
-        events.append("task_one")
-        # Wait indefinitely to simulate long-lived background work, keeping
-        # the task "running" for the assertions below until cleanup cancels it.
-        await asyncio.Event().wait()
-
-    async def task_two(datasette):
-        events.append("task_two")
-        await asyncio.Event().wait()
-
-    class TwoTaskPlugin:
-        __name__ = "TwoTaskPlugin"
-
-        @hookimpl
-        def startup(self, datasette):
-            async def inner():
-                datasette.add_background_task(task_one, name="task-one")
-                datasette.add_background_task(task_two, name="task-two")
-
-            return inner
-
-    ds = Datasette(memory=True)
-    pm.register(TwoTaskPlugin(), name="two_task_plugin")
-    try:
-        app = ds.app()
-        messages = await _drive_lifespan_startup(app)
-        assert {"type": "lifespan.startup.complete"} in messages
-
-        handles = ds._background_tasks.tasks()
-        assert [h.name for h in handles] == ["task-one", "task-two"]
-
-        # Let both tasks run their first line of code.
-        await asyncio.sleep(0)
-        assert handles[0].state == "running"
-        assert handles[1].state == "running"
-        assert events == ["task_one", "task_two"]
-    finally:
-        pm.unregister(name="two_task_plugin")
-        await ds._background_tasks.cancel_all(grace=1.0)
-
-
-@pytest.mark.asyncio
-async def test_launch_waits_for_every_startup_hook_before_running_any_task():
-    # PluginA registers a task from its startup hook; PluginB does the
-    # same from ITS startup hook, which runs after PluginA's (forced with
-    # tryfirst=True on A). Even though A's registration happens first,
-    # A's task body must not actually execute until every startup hook -
-    # including B's - has finished, since launch only happens after
-    # invoke_startup() completes. This is the ordering guarantee that
-    # dissolves datasette-cron's tryfirst=True launch hack.
-    hook_call_order = []
-    seen_names_when_a_ran = {}
-
-    async def task_a(datasette):
-        seen_names_when_a_ran["names"] = [
-            h.name for h in datasette._background_tasks.tasks()
-        ]
-
-    async def task_b(datasette):
-        pass
-
-    class PluginA:
-        __name__ = "PluginA"
-
-        @hookimpl(tryfirst=True)
-        def startup(self, datasette):
-            async def inner():
-                hook_call_order.append("A")
-                datasette.add_background_task(task_a, name="task-a")
-
-            return inner
-
-    class PluginB:
-        __name__ = "PluginB"
-
-        @hookimpl
-        def startup(self, datasette):
-            async def inner():
-                hook_call_order.append("B")
-                datasette.add_background_task(task_b, name="task-b")
-
-            return inner
-
-    ds = Datasette(memory=True)
-    pm.register(PluginA(), name="plugin_a")
-    pm.register(PluginB(), name="plugin_b")
-    try:
-        await ds.start_background_tasks()
-        # Confirm A's startup hook really did run (and register task-a)
-        # strictly before B's startup hook ran.
-        assert hook_call_order == ["A", "B"]
-
-        handles = ds._background_tasks.tasks()
-        await asyncio.wait_for(asyncio.gather(*[h.task for h in handles]), timeout=5)
-        # Yet by the time task-a's own body executed (after launch, which
-        # only happens once every startup hook - including B's - has
-        # finished), task-b was already registered.
-        assert "task-b" in seen_names_when_a_ran["names"]
-    finally:
-        pm.unregister(name="plugin_a")
-        pm.unregister(name="plugin_b")
-
-
-@pytest.mark.asyncio
-async def test_concurrent_first_requests_launch_background_tasks_exactly_once():
-    launch_count = {"n": 0}
-
-    async def counting_task(datasette):
-        launch_count["n"] += 1
-
-    class CountingTaskPlugin:
-        __name__ = "CountingTaskPlugin"
-
-        @hookimpl
-        def startup(self, datasette):
-            async def inner():
-                datasette.add_background_task(counting_task, name="counting-task")
-
-            return inner
-
-    ds = Datasette(memory=True)
-    pm.register(CountingTaskPlugin(), name="counting_task_plugin")
-    try:
-        app = ds.app()
-        transport = httpx2.ASGITransport(app=app)
-        async with httpx2.AsyncClient(
-            transport=transport, base_url="http://localhost"
-        ) as client:
-            responses = await asyncio.gather(
-                *[client.get("/-/versions.json") for _ in range(10)]
-            )
-        assert all(response.status_code == 200 for response in responses)
-
-        handles = ds._background_tasks.tasks()
-        assert len(handles) == 1
-        await asyncio.wait_for(handles[0].task, timeout=5)
-        assert launch_count["n"] == 1
-    finally:
-        pm.unregister(name="counting_task_plugin")
-
-
-@pytest.mark.asyncio
-async def test_post_launch_registration_starts_immediately_and_cancel_works():
-    ds = Datasette(memory=True)
-    await ds.start_background_tasks()  # nothing registered yet, but launched
-
-    started = asyncio.Event()
-
-    async def long_running(datasette):
-        started.set()
-        await asyncio.Event().wait()
-
-    handle = ds.add_background_task(long_running, name="dynamic-task")
-    # Registered after launch: starts immediately rather than sitting in
-    # "registered" limbo.
-    assert handle.state == "running"
-    assert handle.task is not None
-
-    await asyncio.wait_for(started.wait(), timeout=5)
-    assert handle.state == "running"
-
-    handle.cancel()
-    with pytest.raises(asyncio.CancelledError):
-        await handle.task
-    await asyncio.sleep(0)
-    assert handle.state == "cancelled"
-
-
-@pytest.mark.asyncio
-async def test_pre_launch_registration_starts_as_registered():
-    ds = Datasette(memory=True)
-
-    async def task(datasette):
-        pass
-
-    handle = ds.add_background_task(task, name="buffered-task")
-    assert handle.state == "registered"
-    assert handle.task is None
-
-    handle.cancel()  # not yet launched: deregisters instead of cancelling
-    assert handle not in ds._background_tasks.tasks()
-
-
-@pytest.mark.asyncio
-async def test_crashing_task_logs_traceback_and_state_is_crashed(caplog):
-    ds = Datasette(memory=True)
-    await ds.start_background_tasks()
-
-    survivor_ran = asyncio.Event()
-
-    async def crashing_task(datasette):
-        raise RuntimeError("kaboom")
-
-    async def survivor(datasette):
-        survivor_ran.set()
-
-    with caplog.at_level(logging.ERROR, logger="datasette.background_tasks"):
-        crash_handle = ds.add_background_task(crashing_task, name="crashing_task")
-        survivor_handle = ds.add_background_task(survivor, name="survivor")
-        await asyncio.wait_for(
-            asyncio.gather(
-                crash_handle.task, survivor_handle.task, return_exceptions=True
-            ),
-            timeout=5,
-        )
-
-    assert crash_handle.state == "crashed"
-    assert isinstance(crash_handle.exception, RuntimeError)
-    assert str(crash_handle.exception) == "kaboom"
-
-    # The crash must not affect any other task.
-    assert survivor_ran.is_set()
-    assert survivor_handle.state == "completed"
-
-    assert "crashing_task" in caplog.text
-    assert "kaboom" in caplog.text
-    assert "Traceback" in caplog.text
-    assert "RuntimeError" in caplog.text
-
-
-def test_name_collisions_get_suffixed_and_explicit_names_are_respected():
-    ds = Datasette(memory=True)
-
-    async def noop(datasette):
-        pass
-
-    async def another_noop(datasette):
-        pass
-
-    h1 = ds.add_background_task(noop, name="dup")
-    h2 = ds.add_background_task(another_noop, name="dup")
-    h3 = ds.add_background_task(noop, name="dup")
-    assert [h1.name, h2.name, h3.name] == ["dup", "dup-2", "dup-3"]
-
-    h_explicit = ds.add_background_task(noop, name="explicit-name")
-    assert h_explicit.name == "explicit-name"
-
-    h_default = ds.add_background_task(noop)
-    assert h_default.name == noop.__qualname__
-
-
-@pytest.mark.asyncio
-async def test_start_background_tasks_on_bare_datasette():
-    # The headless-CLI path (datasette-rss's `fetch --due` shape): no
-    # server, no lifespan, no first HTTP request - just an explicit call.
-    ran = asyncio.Event()
-
-    async def task(datasette):
-        ran.set()
-
-    ds = Datasette([])
-    assert ds._startup_invoked is False
-
-    handle = ds.add_background_task(task, name="headless-task")
-    assert handle.state == "registered"
-
-    await ds.start_background_tasks()
-
-    assert ds._startup_invoked is True
-    await asyncio.wait_for(ran.wait(), timeout=5)
-    await asyncio.wait_for(handle.task, timeout=5)
-    # handle.task being done only guarantees the coroutine has returned,
-    # not that our done-callback (which updates handle.state) has run yet -
-    # asyncio schedules done-callbacks via call_soon, and awaiting an
-    # already-done future/task returns immediately without giving the loop
-    # a chance to drain its ready queue. Yield once to let it run.
-    await asyncio.sleep(0)
-    assert handle.state == "completed"
-
-
-@pytest.mark.asyncio
-async def test_cancel_all_cancels_running_tasks_and_leaves_completed_alone():
-    ds = Datasette(memory=True)
-    await ds.start_background_tasks()
-
-    async def forever(datasette):
-        await asyncio.Event().wait()
-
-    async def quick(datasette):
-        return "done"
-
-    forever_handle = ds.add_background_task(forever, name="forever")
-    quick_handle = ds.add_background_task(quick, name="quick")
-    await asyncio.wait_for(quick_handle.task, timeout=5)
-    assert quick_handle.state == "completed"
-
-    await ds._background_tasks.cancel_all(grace=1.0)
-
-    assert forever_handle.state == "cancelled"
-    assert quick_handle.state == "completed"
-
-
-@pytest.mark.asyncio
-async def test_cancel_all_logs_stragglers_that_outlive_the_grace_period(caplog):
-    ds = Datasette(memory=True)
-    await ds.start_background_tasks()
-
-    async def stubborn(datasette):
-        with contextlib.suppress(asyncio.CancelledError):
-            await asyncio.sleep(10)
-        # Swallowing CancelledError above and returning normally simulates
-        # a task that ignores cancellation for longer than the grace period.
-        await asyncio.sleep(10)
-
-    handle = ds.add_background_task(stubborn, name="stubborn-task")
-    # Let the task actually start running and reach its first sleep (inside
-    # the CancelledError-suppressing block) before cancelling it - a task
-    # cancelled before it has ever run its first step never enters that
-    # block at all (the throw happens before the coroutine body starts),
-    # so it would finish cancelling immediately instead of behaving like a
-    # straggler.
-    await asyncio.sleep(0)
-
-    with caplog.at_level(logging.WARNING, logger="datasette.background_tasks"):
-        await ds._background_tasks.cancel_all(grace=0.1)
-
-    assert "stubborn-task" in caplog.text
-
-    # Clean up: actually cancel it now that the test has made its
-    # assertion, so it doesn't leak past the end of the test.
-    handle.task.cancel()
-    with contextlib.suppress(asyncio.CancelledError):
-        await handle.task

tests/test_cli_serve_get.py

@@ -52,56 +52,6 @@ def test_serve_with_get(tmp_path_factory):
     pm.unregister(to_unregister)
 
 
-def test_serve_with_get_does_not_launch_background_tasks(tmp_path_factory):
-    # --get must never launch background tasks, even though its TestClient
-    # request
-    # flows through the full ASGI stack (including the AsgiRunOnFirstRequest
-    # fallback that would otherwise launch them). The plugin's startup hook
-    # itself still runs (registration happens) - only the launch is
-    # suppressed, so the sentinel file the background task would write must
-    # never appear.
-    plugins_dir = tmp_path_factory.mktemp("plugins_for_get_background_tasks")
-    sentinel = plugins_dir / "sentinel.txt"
-    (plugins_dir / "bg_task_for_get.py").write_text(
-        textwrap.dedent(
-            f"""
-        from datasette import hookimpl
-
-        @hookimpl
-        def startup(datasette):
-            async def inner():
-                async def task(datasette):
-                    with open("{sentinel!s}", "w") as fp:
-                        fp.write("ran")
-
-                datasette.add_background_task(task, name="get-sentinel-task")
-
-            return inner
-    """,
-        ),
-        "utf-8",
-    )
-    runner = CliRunner()
-    result = runner.invoke(
-        cli,
-        [
-            "serve",
-            "--memory",
-            "--plugins-dir",
-            str(plugins_dir),
-            "--get",
-            "/_memory/-/query.json?sql=select+1",
-        ],
-    )
-    assert result.exit_code == 0, result.output
-    assert not sentinel.exists()
-
-    to_unregister = next(
-        p for p in pm.get_plugins() if p.__name__ == "bg_task_for_get.py"
-    )
-    pm.unregister(to_unregister)
-
-
 def test_serve_with_get_headers():
     runner = CliRunner()
     result = runner.invoke(

tests/test_cli_serve_server.py

@@ -1,15 +1,12 @@
-import signal
 import socket
-import subprocess
-import time
 
-import httpx2
+import httpx
 import pytest
 
 
 @pytest.mark.serial
 def test_serve_localhost_http(ds_localhost_http_server):
-    response = httpx2.get("http://localhost:8041/_memory.json")
+    response = httpx.get("http://localhost:8041/_memory.json")
     assert {
         "database": "_memory",
         "path": "/_memory",
@@ -23,205 +20,11 @@ def test_serve_localhost_http(ds_localhost_http_server):
 )
 def test_serve_unix_domain_socket(ds_unix_domain_socket_server):
     _, uds = ds_unix_domain_socket_server
-    transport = httpx2.HTTPTransport(uds=uds)
+    transport = httpx.HTTPTransport(uds=uds)
-    with httpx2.Client(transport=transport) as client:
+    client = httpx.Client(transport=transport)
     response = client.get("http://localhost/_memory.json")
     assert {
         "database": "_memory",
         "path": "/_memory",
         "tables": [],
     }.items() <= response.json().items()
-
-
-# Shaped after datasette-litestream's startup hook, which schedules a
-# background task with asyncio.get_running_loop().create_task(...):
-# https://github.com/datasette/datasette-litestream
-MARKER_TASK_PLUGIN = """
-import asyncio
-from datasette import hookimpl
-from datasette.utils.asgi import Response
-
-
-@hookimpl
-def startup(datasette):
-    datasette._startup_calls = getattr(datasette, "_startup_calls", 0) + 1
-
-    async def _mark():
-        # Must await before setting the flag: a task with no internal
-        # await point could finish on the throwaway loop before it
-        # closed, masking the regression this test guards against.
-        await asyncio.sleep(0.2)
-        datasette._marker_task_ran = True
-
-    asyncio.get_running_loop().create_task(_mark())
-
-
-@hookimpl
-def register_routes():
-    async def marker_status(datasette):
-        return Response.json(
-            {
-                "marker_task_ran": getattr(datasette, "_marker_task_ran", False),
-                "startup_calls": getattr(datasette, "_startup_calls", 0),
-            }
-        )
-
-    return [(r"^/-/marker-task-ran$", marker_status)]
-"""
-
-
-STARTUP_ERROR_PLUGIN = """
-from datasette import hookimpl
-from datasette.utils import StartupError
-
-
-@hookimpl
-def startup(datasette):
-    raise StartupError("boom from plugin")
-"""
-
-
-@pytest.mark.serial
-def test_startup_hook_background_task_runs_on_serving_loop(serve_with_plugins):
-    """
-    Litestream-shaped regression test: a startup hook that does
-    asyncio.get_running_loop().create_task(...) must have that task
-    actually execute before/while the server is handling requests. This
-    only holds if invoke_startup() and uvicorn.Server.serve() share one
-    event loop. This test fails against unmodified main, where
-    invoke_startup() runs on a throwaway loop that is closed before
-    uvicorn opens its own loop to serve.
-    """
-    _, port = serve_with_plugins({"marker_task_plugin": MARKER_TASK_PLUGIN})
-    # The fixture has already waited for the server to answer requests. The
-    # marker task deliberately awaits before setting its flag, so poll for a
-    # moment rather than assuming it landed before the first request arrived.
-    deadline = time.time() + 3.0
-    payload = {}
-    while time.time() < deadline:
-        payload = httpx2.get(
-            f"http://127.0.0.1:{port}/-/marker-task-ran", timeout=1.0
-        ).json()
-        if payload["marker_task_ran"]:
-            break
-        time.sleep(0.05)
-    assert payload.get("marker_task_ran"), (
-        "The startup hook's asyncio.create_task(...) never ran - "
-        "invoke_startup() and the server are not sharing an event loop"
-    )
-    # Polling above means this test would also pass if the startup hook were
-    # re-run on the serving loop by the first-request fallback - which would
-    # hide exactly the bug being tested. invoke_startup() is idempotent today
-    # so that cannot happen; assert it explicitly so that if the idempotency
-    # guard is ever removed this test fails loudly instead of silently
-    # becoming a no-op.
-    assert payload["startup_calls"] == 1, (
-        "startup hook ran {} times - the marker may have been set by a "
-        "re-run on the serving loop rather than by the original task".format(
-            payload["startup_calls"]
-        )
-    )
-
-
-@pytest.mark.serial
-def test_startup_error_fails_fast_before_port_binds(serve_with_plugins):
-    """
-    A "startup" plugin hook that raises StartupError must fail fast: print
-    the message, exit non-zero, and never accept a connection on the port -
-    the failure must happen before uvicorn.Server binds the socket.
-    """
-    proc, port = serve_with_plugins(
-        {"startup_error_plugin": STARTUP_ERROR_PLUGIN}, wait_for_startup=False
-    )
-    stdout, _ = proc.communicate(timeout=15)
-    output = stdout.decode("utf-8")
-    assert proc.returncode not in (0, None), output
-    assert "boom from plugin" in output, output
-
-    # Nothing is listening on the port now the process has exited. This
-    # confirms the socket was not left bound; on its own it cannot prove the
-    # failure preceded the bind, since a port nothing ever touched also
-    # refuses connections.
-    with (
-        pytest.raises(OSError),
-        socket.create_connection(("127.0.0.1", port), timeout=0.2),
-    ):
-        pass
-
-
-# Verify that SIGTERM and SIGINT sent to `datasette serve` trigger uvicorn's
-# lifespan.shutdown event and run the plugin shutdown hooks. The plugin below
-# writes a sentinel file from its shutdown hook so the tests can check that
-# cleanup ran after the server subprocess exits.
-SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE = """
-import pathlib
-from datasette import hookimpl
-
-SENTINEL_PATH = {sentinel_path!r}
-
-
-@hookimpl
-def shutdown(datasette):
-    pathlib.Path(SENTINEL_PATH).write_text("shutdown ran", "utf-8")
-"""
-
-
-def _start_serve_with_shutdown_sentinel(serve_with_plugins, tmp_path):
-    sentinel_path = tmp_path / "shutdown-sentinel.txt"
-    proc, _ = serve_with_plugins(
-        {
-            "shutdown_sentinel_plugin": SHUTDOWN_SENTINEL_PLUGIN_TEMPLATE.format(
-                sentinel_path=str(sentinel_path)
-            )
-        }
-    )
-    return proc, sentinel_path
-
-
-@pytest.mark.serial
-def test_sigterm_runs_shutdown_hooks(serve_with_plugins, tmp_path):
-    ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel(
-        serve_with_plugins, tmp_path
-    )
-    assert not sentinel_path.exists()
-    ds_proc.send_signal(signal.SIGTERM)
-    try:
-        ds_proc.wait(timeout=10)
-    except subprocess.TimeoutExpired:
-        ds_proc.kill()
-        ds_proc.wait()
-        raise AssertionError(
-            "datasette serve did not exit within 10s of SIGTERM\n"
-            + ds_proc.stdout.read().decode("utf-8")
-        )
-    output = ds_proc.stdout.read().decode("utf-8")
-    assert sentinel_path.exists(), (
-        "shutdown hook never wrote its sentinel file after SIGTERM\n" + output
-    )
-    assert sentinel_path.read_text("utf-8") == "shutdown ran"
-
-
-@pytest.mark.serial
-@pytest.mark.skipif(
-    not hasattr(signal, "SIGINT"), reason="Requires signal.SIGINT support"
-)
-def test_sigint_runs_shutdown_hooks(serve_with_plugins, tmp_path):
-    ds_proc, sentinel_path = _start_serve_with_shutdown_sentinel(
-        serve_with_plugins, tmp_path
-    )
-    assert not sentinel_path.exists()
-    ds_proc.send_signal(signal.SIGINT)
-    try:
-        ds_proc.wait(timeout=10)
-    except subprocess.TimeoutExpired:
-        ds_proc.kill()
-        ds_proc.wait()
-        raise AssertionError(
-            "datasette serve did not exit within 10s of SIGINT\n"
-            + ds_proc.stdout.read().decode("utf-8")
-        )
-    output = ds_proc.stdout.read().decode("utf-8")
-    assert sentinel_path.exists(), (
-        "shutdown hook never wrote its sentinel file after SIGINT\n" + output
-    )
-    assert sentinel_path.read_text("utf-8") == "shutdown ran"

tests/test_column_types.py

@@ -31,7 +31,6 @@ def ds_ct(tmp_path_factory):
         "'https://example.com', '{\"key\": \"value\"}')"
     )
     db.commit()
-    db.close()
     ds = Datasette(
         [db_path],
         config={
@@ -71,7 +70,6 @@ def ds_ct_editor_permission(tmp_path_factory):
         "'https://example.com', '{\"key\": \"value\"}')"
     )
     db.commit()
-    db.close()
     ds = Datasette(
         [db_path],
         config={