Release 1.0a31

Refs #2712, #2735, #2742, #2743, #2747, #2748

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+omit = datasette/_version.py, datasette/utils/shutil_backport.py

.dockerignore

@@ -3,10 +3,11 @@
 .eggs
 .gitignore
 .ipynb_checkpoints
-.travis.yml
 build
 *.spec
 *.egg-info
 dist
 scratchpad
 venv
+*.db
+*.sqlite

.git-blame-ignore-revs

@@ -0,0 +1,4 @@
+# Applying Black
+35d6ee2790e41e96f243c1ff58be0c9c0519a8ce
+368638555160fb9ac78f462d0f79b1394163fa30
+2b344f6a34d2adaa305996a1a580ece06397f6e4

.gitattributes

@@ -1,2 +1 @@
-datasette/_version.py export-subst
 datasette/static/codemirror-* linguist-vendored

.github/FUNDING.yml

@@ -0,0 +1 @@
+github: [simonw]

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+updates:
+- package-ecosystem: pip
+  directory: "/"
+  schedule:
+    interval: daily
+    time: "13:00"
+  groups:
+    python-packages:
+      patterns:
+        - "*"

.github/workflows/deploy-latest.yml

@@ -1,47 +1,132 @@
 name: Deploy latest.datasette.io
 
 on:
+  workflow_dispatch:
   push:
     branches:
-      - master
+    - main
+    #   - 1.0-dev
+
+permissions:
+  contents: read
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
     - name: Check out datasette
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6
     - name: Set up Python
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v6
       with:
-        python-version: 3.8
-    - uses: actions/cache@v1
-      name: Configure pip caching
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
+        python-version: "3.13"
+        cache: pip
     - name: Install Python dependencies
       run: |
-        python -m pip install -e .[test]
+        python -m pip install --upgrade pip
+        python -m pip install . --group dev
+        python -m pip install sphinx-to-sqlite==0.1a1
     - name: Run tests
-      run: pytest
-    - name: Build fixtures.db
-      run: python tests/fixtures.py fixtures.db fixtures.json
-    - name: Set up Cloud Run
-      uses: GoogleCloudPlatform/github-actions/setup-gcloud@master
+      if: ${{ github.ref == 'refs/heads/main' }}
+      run: |
+        pytest -n auto -m "not serial"
+        pytest -m "serial"
+    - name: Build fixtures.db and other files needed to deploy the demo
+      run: |-
+        python tests/fixtures.py \
+          fixtures.db \
+          fixtures-config.json \
+          fixtures-metadata.json \
+          plugins \
+          --extra-db-filename extra_database.db
+    - name: Build docs.db
+      if: ${{ github.ref == 'refs/heads/main' }}
+      run: |-
+        cd docs
+        DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
+        sphinx-to-sqlite ../docs.db _build
+        cd ..
+    - name: Set up the alternate-route demo
+      run: |
+        echo '
+        from datasette import hookimpl
+
+        @hookimpl
+        def startup(datasette):
+            db = datasette.get_database("fixtures2")
+            db.route = "alternative-route"
+        ' > plugins/alternative_route.py
+        cp fixtures.db fixtures2.db
+    - name: And the counters writable stored query demo
+      run: |
+        cat > plugins/counters.py <<EOF
+        from datasette import hookimpl
+        @hookimpl
+        def startup(datasette):
+            db = datasette.add_memory_database("counters")
+            async def inner():
+                await db.execute_write("create table if not exists counters (name text primary key, value integer)")
+                await db.execute_write("insert or ignore into counters (name, value) values ('counter_a', 0)")
+                await db.execute_write("insert or ignore into counters (name, value) values ('counter_b', 0)")
+                await db.execute_write("insert or ignore into counters (name, value) values ('counter_c', 0)")
+                for name in ("counter_a", "counter_b", "counter_c"):
+                    await datasette.add_query(
+                        "counters",
+                        "increment_{}".format(name),
+                        "update counters set value = value + 1 where name = '{}'".format(name),
+                        on_success_message_sql="select 'Counter {name} incremented to ' || value from counters where name = '{name}'".format(name=name),
+                        is_write=True,
+                        is_trusted=True,
+                    )
+                    await datasette.add_query(
+                        "counters",
+                        "decrement_{}".format(name),
+                        "update counters set value = value - 1 where name = '{}'".format(name),
+                        on_success_message_sql="select 'Counter {name} decremented to ' || value from counters where name = '{name}'".format(name=name),
+                        is_write=True,
+                        is_trusted=True,
+                    )
+            return inner
+        EOF
+    # - name: Make some modifications to metadata.json
+    #   run: |
+    #     cat fixtures.json | \
+    #       jq '.databases |= . + {"ephemeral": {"allow": {"id": "*"}}}' | \
+    #       jq '.plugins |= . + {"datasette-ephemeral-tables": {"table_ttl": 900}}' \
+    #       > metadata.json
+    #     cat metadata.json
+    - id: auth
+      name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v3
       with:
-        version: '275.0.0'
-        service_account_email: ${{ secrets.GCP_SA_EMAIL }}
-        service_account_key: ${{ secrets.GCP_SA_KEY }}
+        credentials_json: ${{ secrets.GCP_SA_KEY }}
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v3
     - name: Deploy to Cloud Run
+      env:
+        LATEST_DATASETTE_SECRET: ${{ secrets.LATEST_DATASETTE_SECRET }}
       run: |-
         gcloud config set run/region us-central1
         gcloud config set project datasette-222320
-        datasette publish cloudrun fixtures.db \
-            -m fixtures.json \
+        export SUFFIX="-${GITHUB_REF#refs/heads/}"
+        export SUFFIX=${SUFFIX#-main}
+        # Replace 1.0 with one-dot-zero in SUFFIX
+        export SUFFIX=${SUFFIX//1.0/one-dot-zero}
+        datasette publish cloudrun fixtures.db fixtures2.db extra_database.db \
+            -m fixtures-metadata.json \
+            --plugins-dir=plugins \
             --branch=$GITHUB_SHA \
             --version-note=$GITHUB_SHA \
-            --extra-options="--config template_debug:1" \
-            --service=datasette-latest
+            --extra-options="--setting template_debug 1 --setting trace_debug 1 --crossdb --root" \
+            --install 'datasette-ephemeral-tables>=0.2.2' \
+            --service "datasette-latest$SUFFIX" \
+            --secret $LATEST_DATASETTE_SECRET
+    - name: Deploy to docs as well (only for main)
+      if: ${{ github.ref == 'refs/heads/main' }}
+      run: |-
+        # Deploy docs.db to a different service
+        datasette publish cloudrun docs.db \
+            --branch=$GITHUB_SHA \
+            --version-note=$GITHUB_SHA \
+            --extra-options="--setting template_debug 1" \
+            --service=datasette-docs-latest

.github/workflows/documentation-links.yml

@@ -0,0 +1,16 @@
+name: Read the Docs Pull Request Preview
+on:
+  pull_request:
+    types:
+      - opened
+
+permissions:
+  pull-requests: write
+
+jobs:
+  documentation-links:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: readthedocs/actions/preview@v1
+        with:
+          project-slug: "datasette"

.github/workflows/prettier.yml

@@ -0,0 +1,25 @@
+name: Check JavaScript for conformance with Prettier
+
+on: [push]
+
+permissions:
+  contents: read
+
+jobs:
+  prettier:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out repo
+      uses: actions/checkout@v6
+    - uses: actions/cache@v5
+      name: Configure npm caching
+      with:
+        path: ~/.npm
+        key: ${{ runner.OS }}-npm-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.OS }}-npm-
+    - name: Install dependencies
+      run: npm ci
+    - name: Run prettier
+      run: |-
+        npm run prettier -- --check

.github/workflows/publish.yml

@@ -0,0 +1,109 @@
+name: Publish Python Package
+
+on:
+  release:
+    types: [created]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Install dependencies
+      run: |
+        pip install . --group dev
+    - name: Run tests
+      run: |
+        pytest
+
+  deploy:
+    runs-on: ubuntu-latest
+    needs: [test]
+    environment: release
+    permissions:
+      id-token: write
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.13'
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Install dependencies
+      run: |
+        pip install setuptools wheel build
+    - name: Build
+      run: |
+        python -m build
+    - name: Publish
+      uses: pypa/gh-action-pypi-publish@release/v1
+
+  deploy_static_docs:
+    runs-on: ubuntu-latest
+    needs: [deploy]
+    if: "!github.event.release.prerelease"
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.10'
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Install dependencies
+      run: |
+        python -m pip install . --group dev
+        python -m pip install sphinx-to-sqlite==0.1a1
+    - name: Build docs.db
+      run: |-
+        cd docs
+        DISABLE_SPHINX_INLINE_TABS=1 sphinx-build -b xml . _build
+        sphinx-to-sqlite ../docs.db _build
+        cd ..
+    - id: auth
+      name: Authenticate to Google Cloud
+      uses: google-github-actions/auth@v2
+      with:
+        credentials_json: ${{ secrets.GCP_SA_KEY }}
+    - name: Set up Cloud SDK
+      uses: google-github-actions/setup-gcloud@v3
+    - name: Deploy stable-docs.datasette.io to Cloud Run
+      run: |-
+        gcloud config set run/region us-central1
+        gcloud config set project datasette-222320
+        datasette publish cloudrun docs.db \
+            --service=datasette-docs-stable
+
+  deploy_docker:
+    runs-on: ubuntu-latest
+    needs: [deploy]
+    if: "!github.event.release.prerelease"
+    steps:
+    - uses: actions/checkout@v6
+    - name: Build and push to Docker Hub
+      env:
+        DOCKER_USER: ${{ secrets.DOCKER_USER }}
+        DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
+      run: |-
+        sleep 60 # Give PyPI time to make the new release available
+        docker login -u $DOCKER_USER -p $DOCKER_PASS
+        export REPO=datasetteproject/datasette
+        docker build -f Dockerfile \
+          -t $REPO:${GITHUB_REF#refs/tags/} \
+          --build-arg VERSION=${GITHUB_REF#refs/tags/} .
+        docker tag $REPO:${GITHUB_REF#refs/tags/} $REPO:latest
+        docker push $REPO:${GITHUB_REF#refs/tags/}
+        docker push $REPO:latest

.github/workflows/push_docker_tag.yml

@@ -0,0 +1,28 @@
+name: Push specific Docker tag
+
+on:
+  workflow_dispatch:
+    inputs:
+      version_tag:
+        description: Tag to build and push
+
+permissions:
+  contents: read
+
+jobs:
+  deploy_docker:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - name: Build and push to Docker Hub
+      env:
+        DOCKER_USER: ${{ secrets.DOCKER_USER }}
+        DOCKER_PASS: ${{ secrets.DOCKER_PASS }}
+        VERSION_TAG: ${{ github.event.inputs.version_tag }}
+      run: |-
+        docker login -u $DOCKER_USER -p $DOCKER_PASS
+        export REPO=datasetteproject/datasette
+        docker build -f Dockerfile \
+          -t $REPO:${VERSION_TAG} \
+          --build-arg VERSION=${VERSION_TAG} .
+        docker push $REPO:${VERSION_TAG}

.github/workflows/spellcheck.yml

@@ -0,0 +1,27 @@
+name: Check spelling in documentation
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  spellcheck:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: '**/pyproject.toml'
+    - name: Install dependencies
+      run: |
+        pip install . --group dev
+    - name: Check spelling
+      run: |
+        codespell README.md --ignore-words docs/codespell-ignore-words.txt
+        codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt
+        codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt
+        codespell tests --ignore-words docs/codespell-ignore-words.txt

.github/workflows/stable-docs.yml

@@ -0,0 +1,76 @@
+name: Update Stable Docs
+
+on:
+  release:
+    types: [published]
+  push:
+    branches:
+    - main
+
+permissions:
+  contents: write
+
+jobs:
+  update_stable_docs:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v6
+      with:
+        fetch-depth: 0  # We need all commits to find docs/ changes
+    - name: Set up Git user
+      run: |
+        git config user.name "Automated"
+        git config user.email "actions@users.noreply.github.com"
+    - name: Create stable branch if it does not yet exist
+      run: |
+        if ! git ls-remote --heads origin stable | grep -qE '\bstable\b'; then
+          # Make sure we have all tags locally
+          git fetch --tags --quiet
+
+          # Latest tag that is just numbers and dots (optionally prefixed with 'v')
+          # e.g., 0.65.2 or v0.65.2 — excludes 1.0a20, 1.0-rc1, etc.
+          LATEST_RELEASE=$(
+            git tag -l --sort=-v:refname \
+            | grep -E '^v?[0-9]+(\.[0-9]+){1,3}$' \
+            | head -n1
+          )
+
+          git checkout -b stable
+
+          # If there are any stable releases, copy docs/ from the most recent
+          if [ -n "$LATEST_RELEASE" ]; then
+            rm -rf docs/
+            git checkout "$LATEST_RELEASE" -- docs/ || true
+          fi
+
+          git commit -m "Populate docs/ from $LATEST_RELEASE" || echo "No changes"
+          git push -u origin stable
+        fi
+    - name: Handle Release
+      if: github.event_name == 'release' && !github.event.release.prerelease
+      run: |
+        git fetch --all
+        git checkout stable
+        git reset --hard ${GITHUB_REF#refs/tags/}
+        git push origin stable --force
+    - name: Handle Commit to Main
+      if: contains(github.event.head_commit.message, '!stable-docs')
+      run: |
+        git fetch origin
+        git checkout -b stable origin/stable
+        # Get the list of modified files in docs/ from the current commit
+        FILES=$(git diff-tree --no-commit-id --name-only -r ${{ github.sha }} -- docs/)
+        # Check if the list of files is non-empty
+        if [[ -n "$FILES" ]]; then
+          # Checkout those files to the stable branch to over-write with their contents
+          for FILE in $FILES; do
+            git checkout ${{ github.sha }} -- $FILE
+          done
+          git add docs/
+          git commit -m "Doc changes from ${{ github.sha }}"
+          git push origin stable
+        else
+          echo "No changes to docs/ in this commit."
+          exit 0
+        fi

.github/workflows/test-coverage.yml

@@ -0,0 +1,40 @@
+name: Calculate test coverage
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out datasette
+      uses: actions/checkout@v6
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.12'
+        cache: 'pip'
+        cache-dependency-path: '**/pyproject.toml'
+    - name: Install Python dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install . --group dev
+        python -m pip install pytest-cov
+    - name: Run tests
+      run: |-
+        ls -lah
+        cat .coveragerc
+        pytest -m "not serial" --cov=datasette --cov-config=.coveragerc --cov-report xml:coverage.xml --cov-report term -x
+        ls -lah
+    - name: Upload coverage report
+      uses: codecov/codecov-action@v1
+      with:
+        token: ${{ secrets.CODECOV_TOKEN }}
+        file: coverage.xml

.github/workflows/test-pyodide.yml

@@ -0,0 +1,33 @@
+name: Test in Pyodide with shot-scraper
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.10"
+        cache: 'pip'
+        cache-dependency-path: '**/pyproject.toml'
+    - name: Cache Playwright browsers
+      uses: actions/cache@v5
+      with:
+        path: ~/.cache/ms-playwright/
+        key: ${{ runner.os }}-browsers
+    - name: Install Playwright dependencies
+      run: |
+        pip install shot-scraper build
+        shot-scraper install
+    - name: Run test
+      run: |
+        ./test-in-pyodide-with-shot-scraper.sh

.github/workflows/test-sqlite-support.yml

@@ -0,0 +1,53 @@
+name: Test SQLite versions
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ${{ matrix.platform }}
+    continue-on-error: true
+    strategy:
+      matrix:
+        platform: [ubuntu-latest]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        sqlite-version: [
+          #"3", # latest version
+          "3.46",
+          #"3.45",
+          #"3.27",
+          #"3.26",
+          "3.25",
+          #"3.25.3", # 2018-09-25, window functions breaks test_upsert for some reason on 3.10, skip for now
+          #"3.24", # 2018-06-04, added UPSERT support
+          #"3.23.1" # 2018-04-10, before UPSERT
+        ]
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Set up SQLite ${{ matrix.sqlite-version }}
+      uses: asg017/sqlite-versions@71ea0de37ae739c33e447af91ba71dda8fcf22e6
+      with:
+        version: ${{ matrix.sqlite-version }}
+        cflags: "-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_RTREE -DSQLITE_ENABLE_JSON1"
+    - run: python3 -c "import sqlite3; print(sqlite3.sqlite_version)"
+    - run: echo $LD_LIBRARY_PATH
+    - name: Build extension for --load-extension test
+      run: |-
+        (cd tests && gcc ext.c -fPIC -shared -o ext.so)
+    - name: Install dependencies
+      run: |
+        pip install . --group dev
+        pip freeze
+    - name: Run tests
+      run: |
+        pytest -n auto -m "not serial"
+        pytest -m "serial"

.github/workflows/test.yml

@@ -0,0 +1,52 @@
+name: Test
+
+on: [push, pull_request]
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
+        cache: pip
+        cache-dependency-path: pyproject.toml
+    - name: Build extension for --load-extension test
+      run: |-
+        (cd tests && gcc ext.c -fPIC -shared -o ext.so)
+    - name: Install dependencies
+      run: |
+        pip install . --group dev
+        pip freeze
+    - name: Run tests
+      run: |
+        pytest -n auto -m "not serial"
+        pytest -m "serial"
+        # And the test that exceeds a localhost HTTPS server
+        tests/test_datasette_https_server.sh
+    - name: Black
+      run: |
+        black --version
+        black --check .
+    - name: Ruff
+      run: ruff check datasette tests
+    - name: Check if cog needs to be run
+      run: |
+        cog --check docs/*.rst
+    - name: Check if blacken-docs needs to be run
+      run: |
+        # This fails on syntax errors, or a diff was applied
+        blacken-docs -l 60 docs/*.rst
+    - name: Test DATASETTE_LOAD_PLUGINS
+      run: |
+        pip install datasette-init datasette-json-html
+        tests/test-datasette-load-plugins.sh

.github/workflows/tmate-mac.yml

@@ -0,0 +1,15 @@
+name: tmate session mac
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: macos-latest
+    steps:
+    - uses: actions/checkout@v6
+    - name: Setup tmate session
+      uses: mxschmitt/action-tmate@v3

.github/workflows/tmate.yml

@@ -0,0 +1,18 @@
+name: tmate session
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  models: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v6
+    - name: Setup tmate session
+      uses: mxschmitt/action-tmate@v3
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -5,6 +5,12 @@ scratchpad
 
 .vscode
 
+uv.lock
+data.db
+
+# test databases
+*.db
+
 # We don't use Pipfile, so ignore them
 Pipfile
 Pipfile.lock
@@ -116,3 +122,13 @@ ENV/
 
 # macOS files
 .DS_Store
+node_modules
+.*.swp
+
+# In case someone compiled tests/ext.c for test_load_extensions, don't
+# include it in source control.
+tests/*.dylib
+tests/*.so
+tests/*.dll
+
+.idea

.prettierrc

@@ -0,0 +1,4 @@
+{
+  "tabWidth": 2,
+  "useTabs": false
+}

.readthedocs.yaml

@@ -0,0 +1,17 @@
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+build:
+  os: ubuntu-24.04
+  tools:
+    python: "3.13"
+  jobs:
+    install:
+    - pip install --upgrade pip
+    - pip install . --group dev
+
+formats:
+- pdf
+- epub

.travis.yml

@@ -1,43 +0,0 @@
-language: python
-dist: xenial
-
-# 3.6 is listed first so it gets used for the later build stages
-python:
-  - "3.6"
-  - "3.7"
-  - "3.8"
-
-# Executed for 3.5 AND 3.5 as the first "test" stage:
-script:
-  - pip install -U pip wheel
-  - pip install .[test]
-  - pytest
-
-cache:
-    directories:
-        - $HOME/.cache/pip
-
-# This defines further stages that execute after the tests
-jobs:
-  include:
-    - stage: release tagged version
-      if: tag IS present
-      python: 3.6
-      deploy:
-        - provider: pypi
-          user: simonw
-          distributions: bdist_wheel
-          password: ${PYPI_PASSWORD}
-          on:
-            branch: master
-            tags: true
-    - stage: publish docker image
-      if: tag IS present
-      python: 3.6
-      script:
-        # Build and release to Docker Hub
-        - docker login -u $DOCKER_USER -p $DOCKER_PASS
-        - export REPO=datasetteproject/datasette
-        - docker build -f Dockerfile -t $REPO:$TRAVIS_TAG .
-        - docker tag $REPO:$TRAVIS_TAG $REPO:latest
-        - docker push $REPO

CODE_OF_CONDUCT.md

@@ -0,0 +1,128 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+`swillison+datasette-code-of-conduct@gmail.com`.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+https://www.contributor-covenant.org/version/2/0/code_of_conduct.html.
+
+Community Impact Guidelines were inspired by [Mozilla's code of conduct
+enforcement ladder](https://github.com/mozilla/diversity).
+
+[homepage]: https://www.contributor-covenant.org
+
+For answers to common questions about this code of conduct, see the FAQ at
+https://www.contributor-covenant.org/faq. Translations are available at
+https://www.contributor-covenant.org/translations.

Dockerfile

@@ -1,42 +1,18 @@
-FROM python:3.7.2-slim-stretch as build
+FROM python:3.11.0-slim-bullseye as build
 
-# Setup build dependencies
-RUN apt update \
-&& apt install -y python3-dev build-essential wget libxml2-dev libproj-dev libgeos-dev libsqlite3-dev zlib1g-dev pkg-config git \
- && apt clean
+# Version of Datasette to install, e.g. 0.55
+#   docker build . -t datasette --build-arg VERSION=0.55
+ARG VERSION
 
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends libsqlite3-mod-spatialite && \
+    apt clean && \
+    rm -rf /var/lib/apt && \
+    rm -rf /var/lib/dpkg/info/*
 
-RUN wget "https://www.sqlite.org/2020/sqlite-autoconf-3310100.tar.gz" && tar xzf sqlite-autoconf-3310100.tar.gz \
-    && cd sqlite-autoconf-3310100 && ./configure --disable-static --enable-fts5 --enable-json1 CFLAGS="-g -O2 -DSQLITE_ENABLE_FTS3=1 -DSQLITE_ENABLE_FTS4=1 -DSQLITE_ENABLE_RTREE=1 -DSQLITE_ENABLE_JSON1" \
-    && make && make install
-
-RUN wget "https://www.gaia-gis.it/gaia-sins/freexl-1.0.5.tar.gz" && tar zxf freexl-1.0.5.tar.gz \
-    && cd freexl-1.0.5 && ./configure && make && make install
-
-RUN wget "https://www.gaia-gis.it/gaia-sins/libspatialite-4.4.0-RC0.tar.gz" && tar zxf libspatialite-4.4.0-RC0.tar.gz \
-    && cd libspatialite-4.4.0-RC0 && ./configure && make && make install
-
-RUN wget "https://www.gaia-gis.it/gaia-sins/readosm-1.1.0.tar.gz" && tar zxf readosm-1.1.0.tar.gz && cd readosm-1.1.0 && ./configure && make && make install
-
-RUN wget "https://www.gaia-gis.it/gaia-sins/spatialite-tools-4.4.0-RC0.tar.gz" && tar zxf spatialite-tools-4.4.0-RC0.tar.gz \
-    && cd spatialite-tools-4.4.0-RC0 && ./configure && make && make install
-
-
-# Add local code to the image instead of fetching from pypi.
-COPY . /datasette
-
-RUN pip install /datasette
-
-FROM python:3.7.2-slim-stretch
-
-# Copy python dependencies and spatialite libraries
-COPY --from=build /usr/local/lib/ /usr/local/lib/
-# Copy executables
-COPY --from=build /usr/local/bin /usr/local/bin
-# Copy spatial extensions
-COPY --from=build /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu
-
-ENV LD_LIBRARY_PATH=/usr/local/lib
+RUN pip install https://github.com/simonw/datasette/archive/refs/tags/${VERSION}.zip && \
+    find /usr/local/lib -name '__pycache__' | xargs rm -r && \
+    rm -rf /root/.cache/pip
 
 EXPOSE 8001
 CMD ["datasette"]

Justfile

@@ -0,0 +1,60 @@
+export DATASETTE_SECRET := "not_a_secret"
+
+# Run tests and linters
+@default: test lint
+
+# Setup project
+@init:
+  uv sync
+
+# Run pytest with supplied options
+@test *options: init
+  uv run pytest -n auto {{options}}
+
+@codespell:
+  uv run codespell README.md --ignore-words docs/codespell-ignore-words.txt
+  uv run codespell docs/*.rst --ignore-words docs/codespell-ignore-words.txt
+  uv run codespell datasette -S datasette/static --ignore-words docs/codespell-ignore-words.txt
+  uv run codespell tests --ignore-words docs/codespell-ignore-words.txt
+
+# Run linters: black, ruff, cog
+@lint: codespell
+  uv run black datasette tests --check
+  uv run ruff check datasette tests
+  uv run cog --check README.md docs/*.rst
+
+# Apply ruff fixes
+@fix:
+  uv run ruff check --fix datasette tests
+
+# Rebuild docs with cog
+@cog:
+  uv run cog -r README.md docs/*.rst
+
+# Serve live docs on localhost:8000
+@docs: cog blacken-docs
+  uv run make -C docs livehtml
+
+# Build docs as static HTML
+@docs-build: cog blacken-docs
+  rm -rf docs/_build && cd docs && uv run make html
+
+# Apply Black
+@black:
+  uv run black datasette tests
+
+# Apply blacken-docs
+@blacken-docs:
+  uv run blacken-docs -l 60 docs/*.rst
+
+# Apply prettier
+@prettier:
+  npm run fix
+
+# Format code with both black and prettier
+@format: black prettier blacken-docs
+
+@serve *options:
+  uv run sqlite-utils create-database data.db
+  uv run sqlite-utils create-table data.db docs id integer title text --pk id --ignore
+  uv run python -m datasette data.db --root --reload {{options}}

MANIFEST.in

@@ -1,3 +1,5 @@
 recursive-include datasette/static *
+recursive-include datasette/templates *
 include versioneer.py
 include datasette/_version.py
+include LICENSE

README.md

@@ -1,88 +1,42 @@
-# Datasette
+<img src="https://datasette.io/static/datasette-logo.svg" alt="Datasette">
 
 [![PyPI](https://img.shields.io/pypi/v/datasette.svg)](https://pypi.org/project/datasette/)
+[![Changelog](https://img.shields.io/github/v/release/simonw/datasette?label=changelog)](https://docs.datasette.io/en/latest/changelog.html)
 [![Python 3.x](https://img.shields.io/pypi/pyversions/datasette.svg?logo=python&logoColor=white)](https://pypi.org/project/datasette/)
-[![Travis CI](https://travis-ci.org/simonw/datasette.svg?branch=master)](https://travis-ci.org/simonw/datasette)
-[![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](http://datasette.readthedocs.io/en/latest/?badge=latest)
-[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/master/LICENSE)
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://black.readthedocs.io/en/stable/)
+[![Tests](https://github.com/simonw/datasette/workflows/Test/badge.svg)](https://github.com/simonw/datasette/actions?query=workflow%3ATest)
+[![Documentation Status](https://readthedocs.org/projects/datasette/badge/?version=latest)](https://docs.datasette.io/en/latest/?badge=latest)
+[![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette/blob/main/LICENSE)
 [![docker: datasette](https://img.shields.io/badge/docker-datasette-blue)](https://hub.docker.com/r/datasetteproject/datasette)
+[![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://datasette.io/discord)
 
-*A tool for exploring and publishing data*
+*An open source multi-tool for exploring and publishing data*
 
 Datasette is a tool for exploring and publishing data. It helps people take data of any shape or size and publish that as an interactive, explorable website and accompanying API.
 
-Datasette is aimed at data journalists, museum curators, archivists, local governments and anyone else who has data that they wish to share with the world.
+Datasette is aimed at data journalists, museum curators, archivists, local governments, scientists, researchers and anyone else who has data that they wish to share with the world.
 
-[Explore a demo](https://fivethirtyeight.datasettes.com/fivethirtyeight), watch [a video about the project](https://www.youtube.com/watch?v=pTr1uLQTJNE) or try it out by [uploading and publishing your own CSV data](https://simonwillison.net/2019/Apr/23/datasette-glitch/).
+[Explore a demo](https://datasette.io/global-power-plants/global-power-plants), watch [a video about the project](https://simonwillison.net/2021/Feb/7/video/) or try it out [on GitHub Codespaces](https://github.com/datasette/datasette-studio).
 
-* Comprehensive documentation: http://datasette.readthedocs.io/
-* Examples: https://github.com/simonw/datasette/wiki/Datasettes
-* Live demo of current master: https://latest.datasette.io/
+* [datasette.io](https://datasette.io/) is the official project website
+* Latest [Datasette News](https://datasette.io/news)
+* Comprehensive documentation: https://docs.datasette.io/
+* Examples: https://datasette.io/examples
+* Live demo of current `main` branch: https://latest.datasette.io/
+* Questions, feedback or want to talk about the project? Join our [Discord](https://datasette.io/discord)
 
-## News
-
- * 24th March 2020: [Datasette 0.39](http://datasette.readthedocs.io/en/latest/changelog.html#v0-39) - New `base_url` configuration option for running Datasette under a different URL prefix, `"sort"` and `"sort_desc"` metadata options for setting a default sort order for a table.
- * 8th March 2020: [Datasette 0.38](http://datasette.readthedocs.io/en/latest/changelog.html#v0-38) - New `--memory` option for `datasete publish cloudrun`, [Docker image](https://hub.docker.com/r/datasetteproject/datasette) upgraded to SQLite 3.31.1.
- * 25th February 2020: [Datasette 0.37](http://datasette.readthedocs.io/en/latest/changelog.html#v0-37) - new internal APIs enabling plugins to safely write to databases. Read more here: [Datasette Writes](https://simonwillison.net/2020/Feb/26/weeknotes-datasette-writes/).
- * 21st February 2020: [Datasette 0.36](http://datasette.readthedocs.io/en/latest/changelog.html#v0-36) - new internals documentation for plugins, `prepare_connection()` now accepts optional `database` and `datasette` arguments.
- * 4th February 2020: [Datasette 0.35](http://datasette.readthedocs.io/en/latest/changelog.html#v0-35) - new `.render_template()` method for plugins.
- * 29th January 2020: [Datasette 0.34](http://datasette.readthedocs.io/en/latest/changelog.html#v0-34) - improvements to search, `datasette publish cloudrun` and `datasette package`.
- * 21st January 2020: [Deploying a data API using GitHub Actions and Cloud Run](https://simonwillison.net/2020/Jan/21/github-actions-cloud-run/) - how to use GitHub Actions and Google Cloud Run to automatically scrape data and deploy the result as an API with Datasette.
- * 22nd December 2019: [Datasette 0.33](http://datasette.readthedocs.io/en/latest/changelog.html#v0-33) - various small improvements.
- * 19th December 2019: [Building tools to bring data-driven reporting to more newsrooms](https://medium.com/jsk-class-of-2020/building-tools-to-bring-data-driven-reporting-to-more-newsrooms-4520a0c9b3f2) - some notes on my JSK fellowship so far.
- * 2nd December 2019: [Niche Museums](https://www.niche-museums.com/) is a new site entirely powered by Datasette, using custom templates and plugins. [niche-museums.com, powered by Datasette](https://simonwillison.net/2019/Nov/25/niche-museums/) describes how the site works, and [datasette-atom: Define an Atom feed using a custom SQL query](https://simonwillison.net/2019/Dec/3/datasette-atom/) describes how the new [datasette-atom plugin](https://github.com/simonw/datasette-atom) was used to add an Atom syndication feed to the site.
- * 14th November 2019: [Datasette 0.32](https://datasette.readthedocs.io/en/stable/changelog.html#v0-32) now uses asynchronous rendering in Jinja templates, which means template functions can perform asynchronous operations such as executing SQL queries. [datasette-template-sql](https://github.com/simonw/datasette-template-sql) is a new plugin uses this capability to add a new custom `sql(sql_query)` template function.
- * 11th November 2019: [Datasette 0.31](https://datasette.readthedocs.io/en/stable/changelog.html#v0-31) - the first version of Datasette to support Python 3.8, which means dropping support for Python 3.5.
- * 18th October 2019: [Datasette 0.30](https://datasette.readthedocs.io/en/stable/changelog.html#v0-30)
- * 13th July 2019: [Single sign-on against GitHub using ASGI middleware](https://simonwillison.net/2019/Jul/14/sso-asgi/) talks about the implementation of [datasette-auth-github](https://github.com/simonw/datasette-auth-github) in more detail.
- * 7th July 2019: [Datasette 0.29](https://datasette.readthedocs.io/en/stable/changelog.html#v0-29) - ASGI, new plugin hooks, facet by date and much, much more...
-   * [datasette-auth-github](https://github.com/simonw/datasette-auth-github) - a new plugin for Datasette 0.29 that lets you require users to authenticate against GitHub before accessing your Datasette instance. You can whitelist specific users, or you can restrict access to members of specific GitHub organizations or teams.
-   * [datasette-cors](https://github.com/simonw/datasette-cors) - a plugin that lets you configure CORS access from a list of domains (or a set of domain wildcards) so you can make JavaScript calls to a Datasette instance from a specific set of other hosts.
- * 23rd June 2019: [Porting Datasette to ASGI, and Turtles all the way down](https://simonwillison.net/2019/Jun/23/datasette-asgi/)
- * 21st May 2019: The anonymized raw data from [the Stack Overflow Developer Survey 2019](https://stackoverflow.blog/2019/05/21/public-data-release-of-stack-overflows-2019-developer-survey/) has been [published in partnership with Glitch](https://glitch.com/culture/discover-insights-explore-developer-survey-results-2019/), powered by Datasette.
- * 19th May 2019: [Datasette 0.28](https://datasette.readthedocs.io/en/stable/changelog.html#v0-28) - a salmagundi of new features!
-   * No longer immutable! Datasette now supports [databases that change](https://datasette.readthedocs.io/en/stable/changelog.html#supporting-databases-that-change).
-   * [Faceting improvements](https://datasette.readthedocs.io/en/stable/changelog.html#faceting-improvements-and-faceting-plugins) including facet-by-JSON-array and the ability to define custom faceting using plugins.
-   * [datasette publish cloudrun](https://datasette.readthedocs.io/en/stable/changelog.html#datasette-publish-cloudrun) lets you publish databases to Google's new Cloud Run hosting service.
-   * New [register_output_renderer](https://datasette.readthedocs.io/en/stable/changelog.html#register-output-renderer-plugins) plugin hook for adding custom output extensions to Datasette in addition to the default `.json` and `.csv`.
-   * Dozens of other smaller features and tweaks - see [the release notes](https://datasette.readthedocs.io/en/stable/changelog.html#v0-28) for full details.
-   * Read more about this release here: [Datasette 0.28—and why master should always be releasable](https://simonwillison.net/2019/May/19/datasette-0-28/)
- * 24th February 2019: [
-sqlite-utils: a Python library and CLI tool for building SQLite databases](https://simonwillison.net/2019/Feb/25/sqlite-utils/) - a partner tool for easily creating SQLite databases for use with Datasette.
- * 31st Janary 2019: [Datasette 0.27](https://datasette.readthedocs.io/en/latest/changelog.html#v0-27) - `datasette plugins` command, newline-delimited JSON export option, new documentation on [The Datasette Ecosystem](https://datasette.readthedocs.io/en/latest/ecosystem.html).
- * 10th January 2019: [Datasette 0.26.1](http://datasette.readthedocs.io/en/latest/changelog.html#v0-26-1) - SQLite upgrade in Docker image, `/-/versions` now shows SQLite compile options.
- * 2nd January 2019: [Datasette 0.26](http://datasette.readthedocs.io/en/latest/changelog.html#v0-26) - minor bug fixes, `datasette publish now --alias` argument.
-* 18th December 2018: [Fast Autocomplete Search for Your Website](https://24ways.org/2018/fast-autocomplete-search-for-your-website/) - a new tutorial on using Datasette to build a JavaScript autocomplete search engine.
-* 3rd October 2018: [The interesting ideas in Datasette](https://simonwillison.net/2018/Oct/4/datasette-ideas/) - a write-up of some of the less obvious interesting ideas embedded in the Datasette project.
-* 19th September 2018: [Datasette 0.25](http://datasette.readthedocs.io/en/latest/changelog.html#v0-25) - New plugin hooks, improved database view support and an easier way to use more recent versions of SQLite.
-* 23rd July 2018: [Datasette 0.24](http://datasette.readthedocs.io/en/latest/changelog.html#v0-24) - a number of small new features
-* 29th June 2018: [datasette-vega](https://github.com/simonw/datasette-vega), a new plugin for visualizing data as bar, line or scatter charts
-* 21st June 2018: [Datasette 0.23.1](http://datasette.readthedocs.io/en/latest/changelog.html#v0-23-1) - minor bug fixes
-* 18th June 2018: [Datasette 0.23: CSV, SpatiaLite and more](http://datasette.readthedocs.io/en/latest/changelog.html#v0-23) - CSV export, foreign key expansion in JSON and CSV, new config options, improved support for SpatiaLite and a bunch of other improvements
-* 23rd May 2018: [Datasette 0.22.1 bugfix](https://github.com/simonw/datasette/releases/tag/0.22.1) plus we now use [versioneer](https://github.com/warner/python-versioneer)
-* 20th May 2018: [Datasette 0.22: Datasette Facets](https://simonwillison.net/2018/May/20/datasette-facets)
-* 5th May 2018: [Datasette 0.21: New _shape=, new _size=, search within columns](https://github.com/simonw/datasette/releases/tag/0.21)
-* 25th April 2018: [Exploring the UK Register of Members Interests with SQL and Datasette](https://simonwillison.net/2018/Apr/25/register-members-interests/) - a tutorial describing how [register-of-members-interests.datasettes.com](https://register-of-members-interests.datasettes.com/) was built ([source code here](https://github.com/simonw/register-of-members-interests))
-* 20th April 2018: [Datasette plugins, and building a clustered map visualization](https://simonwillison.net/2018/Apr/20/datasette-plugins/) - introducing Datasette's new plugin system and [datasette-cluster-map](https://pypi.org/project/datasette-cluster-map/), a plugin for visualizing data on a map
-* 20th April 2018: [Datasette 0.20: static assets and templates for plugins](https://github.com/simonw/datasette/releases/tag/0.20)
-* 16th April 2018: [Datasette 0.19: plugins preview](https://github.com/simonw/datasette/releases/tag/0.19)
-* 14th April 2018: [Datasette 0.18: units](https://github.com/simonw/datasette/releases/tag/0.18)
-* 9th April 2018: [Datasette 0.15: sort by column](https://github.com/simonw/datasette/releases/tag/0.15)
-* 28th March 2018: [Baltimore Sun Public Salary Records](https://simonwillison.net/2018/Mar/28/datasette-in-the-wild/) - a data journalism project from the Baltimore Sun powered by Datasette - source code [is available here](https://github.com/baltimore-sun-data/salaries-datasette)
-* 27th March 2018: [Cloud-first: Rapid webapp deployment using containers](https://wwwf.imperial.ac.uk/blog/research-software-engineering/2018/03/27/cloud-first-rapid-webapp-deployment-using-containers/) - a tutorial covering deploying Datasette using Microsoft Azure by the Research Software Engineering team at Imperial College London
-* 28th January 2018: [Analyzing my Twitter followers with Datasette](https://simonwillison.net/2018/Jan/28/analyzing-my-twitter-followers/) - a tutorial on using Datasette to analyze follower data pulled from the Twitter API
-* 17th January 2018: [Datasette Publish: a web app for publishing CSV files as an online database](https://simonwillison.net/2018/Jan/17/datasette-publish/)
-* 12th December 2017: [Building a location to time zone API with SpatiaLite, OpenStreetMap and Datasette](https://simonwillison.net/2017/Dec/12/building-a-location-time-zone-api/)
-* 9th December 2017: [Datasette 0.14: customization edition](https://github.com/simonw/datasette/releases/tag/0.14)
-* 25th November 2017: [New in Datasette: filters, foreign keys and search](https://simonwillison.net/2017/Nov/25/new-in-datasette/)
-* 13th November 2017: [Datasette: instantly create and publish an API for your SQLite databases](https://simonwillison.net/2017/Nov/13/datasette/)
+Want to stay up-to-date with the project? Subscribe to the [Datasette newsletter](https://datasette.substack.com/) for tips, tricks and news on what's new in the Datasette ecosystem.
 
 ## Installation
 
-    pip3 install datasette
+If you are on a Mac, [Homebrew](https://brew.sh/) is the easiest way to install Datasette:
 
-Datasette requires Python 3.6 or higher. We also have [detailed installation instructions](https://datasette.readthedocs.io/en/stable/installation.html) covering other options such as Docker.
+    brew install datasette
+
+You can also install it using `pip` or `pipx`:
+
+    pip install datasette
+
+Datasette requires Python 3.8 or higher. We also have [detailed installation instructions](https://docs.datasette.io/en/stable/installation.html) covering other options such as Docker.
 
 ## Basic usage
 
@@ -94,46 +48,12 @@ This will start a web server on port 8001 - visit http://localhost:8001/ to acce
 
 Use Chrome on OS X? You can run datasette against your browser history like so:
 
-     datasette ~/Library/Application\ Support/Google/Chrome/Default/History
+     datasette ~/Library/Application\ Support/Google/Chrome/Default/History --nolock
 
 Now visiting http://localhost:8001/History/downloads will show you a web interface to browse your downloads data:
 
 ![Downloads table rendered by datasette](https://static.simonwillison.net/static/2017/datasette-downloads.png)
 
-## datasette serve options
-
-    Usage: datasette serve [OPTIONS] [FILES]...
-
-      Serve up specified SQLite database files with a web UI
-
-    Options:
-      -i, --immutable PATH      Database files to open in immutable mode
-      -h, --host TEXT           Host for server. Defaults to 127.0.0.1 which means
-                                only connections from the local machine will be
-                                allowed. Use 0.0.0.0 to listen to all IPs and
-                                allow access from other machines.
-      -p, --port INTEGER        Port for server, defaults to 8001
-      --debug                   Enable debug mode - useful for development
-      --reload                  Automatically reload if database or code change
-                                detected - useful for development
-      --cors                    Enable CORS by serving Access-Control-Allow-
-                                Origin: *
-      --load-extension PATH     Path to a SQLite extension to load
-      --inspect-file TEXT       Path to JSON file created using "datasette
-                                inspect"
-      -m, --metadata FILENAME   Path to JSON file containing license/source
-                                metadata
-      --template-dir DIRECTORY  Path to directory containing custom templates
-      --plugins-dir DIRECTORY   Path to directory containing custom plugins
-      --static STATIC MOUNT     mountpoint:path-to-directory for serving static
-                                files
-      --memory                  Make :memory: database available
-      --config CONFIG           Set config option using configname:value
-                                datasette.readthedocs.io/en/latest/config.html
-      --version-note TEXT       Additional note to show on /-/versions
-      --help-config             Show available config options
-      --help                    Show this message and exit.
-
 ## metadata.json
 
 If you want to include licensing and source information in the generated datasette website you can do so using a JSON file that looks something like this:
@@ -164,4 +84,8 @@ Or:
 
 This will create a docker image containing both the datasette application and the specified SQLite database files. It will then deploy that image to Heroku or Cloud Run and give you a URL to access the resulting website and API.
 
-See [Publishing data](https://datasette.readthedocs.io/en/stable/publish.html) in the documentation for more details.
+See [Publishing data](https://docs.datasette.io/en/stable/publish.html) in the documentation for more details.
+
+## Datasette Lite
+
+[Datasette Lite](https://lite.datasette.io/) is Datasette packaged using WebAssembly so that it runs entirely in your browser, no Python web application server required. Read more about that in the [Datasette Lite documentation](https://github.com/simonw/datasette-lite/blob/main/README.md).

_config.yml

@@ -1 +0,0 @@
-theme: jekyll-theme-architect

codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true

datasette/__init__.py

@@ -1,3 +1,9 @@
+from datasette.permissions import Permission  # noqa
 from datasette.version import __version_info__, __version__  # noqa
+from datasette.events import Event  # noqa
+from datasette.tokens import TokenHandler, TokenRestrictions  # noqa
+from datasette.utils.asgi import Forbidden, NotFound, Request, Response  # noqa
+from datasette.utils import actor_matches_allow  # noqa
+from datasette.views import Context  # noqa
 from .hookspecs import hookimpl  # noqa
 from .hookspecs import hookspec  # noqa

datasette/_pytest_plugin.py

@@ -0,0 +1,108 @@
+"""
+Pytest plugin that automatically closes any Datasette instances constructed
+during a pytest test — both in the test body and in function-scoped
+fixtures. Instances constructed by session-, module-, class- or package-
+scoped fixtures are left alone, because other tests in the session will
+still want to use them.
+
+Registered as a pytest11 entry point in pyproject.toml so that downstream
+projects using Datasette get the same FD-safety net for their own tests.
+
+Opt out by setting ``datasette_autoclose = false`` in pytest.ini (or the
+equivalent ini file).
+"""
+
+from __future__ import annotations
+
+import contextvars
+import weakref
+
+import pytest
+
+from datasette.app import Datasette
+
+_active_instances: contextvars.ContextVar[list | None] = contextvars.ContextVar(
+    "datasette_active_instances", default=None
+)
+
+_original_init = Datasette.__init__
+
+
+def _tracking_init(self, *args, **kwargs):
+    _original_init(self, *args, **kwargs)
+    instances = _active_instances.get()
+    if instances is not None:
+        instances.append(weakref.ref(self))
+
+
+Datasette.__init__ = _tracking_init
+
+
+def pytest_addoption(parser):
+    parser.addini(
+        "datasette_autoclose",
+        help=(
+            "Automatically close Datasette instances created inside test "
+            "bodies and function-scoped fixtures (default: true)."
+        ),
+        default="true",
+    )
+
+
+def _enabled(config) -> bool:
+    value = config.getini("datasette_autoclose")
+    if isinstance(value, bool):
+        return value
+    return str(value).strip().lower() not in ("false", "0", "no", "off")
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_protocol(item, nextitem):
+    """Track Datasette instances across setup, call and teardown; close at end."""
+    if not _enabled(item.config):
+        yield
+        return
+    refs: list[weakref.ref] = []
+    token = _active_instances.set(refs)
+    try:
+        yield
+    finally:
+        _active_instances.reset(token)
+        for ref in reversed(refs):
+            ds = ref()
+            if ds is None:
+                continue
+            try:
+                ds.close()
+            except Exception as e:
+                item.warn(
+                    pytest.PytestUnraisableExceptionWarning(
+                        f"Error closing Datasette instance: {e!r}"
+                    )
+                )
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_fixture_setup(fixturedef, request):
+    """Exempt instances created by non-function-scoped fixtures.
+
+    Session-, module-, class- and package-scoped fixtures produce Datasette
+    instances that must survive beyond the current test — other tests in
+    the session will still use them. When such a fixture creates one or
+    more Datasette instances during its setup, we snapshot the tracking
+    list before the fixture runs and subtract off any instances that were
+    added during its setup, so they don't get closed at test teardown.
+    """
+    refs = _active_instances.get()
+    if refs is None:
+        yield
+        return
+    before_ids = {id(ref) for ref in refs}
+    yield
+    if fixturedef.scope != "function":
+        new_refs = [ref for ref in refs if id(ref) not in before_ids]
+        for new_ref in new_refs:
+            try:
+                refs.remove(new_ref)
+            except ValueError:
+                pass

datasette/_version.py

@@ -1,556 +0,0 @@
-# This file helps to compute a version number in source trees obtained from
-# git-archive tarball (such as those provided by githubs download-from-tag
-# feature). Distribution tarballs (built by setup.py sdist) and build
-# directories (produced by setup.py build) will contain a much shorter file
-# that just contains the computed version number.
-
-# This file is released into the public domain. Generated by
-# versioneer-0.18 (https://github.com/warner/python-versioneer)
-
-"""Git implementation of _version.py."""
-
-import errno
-import os
-import re
-import subprocess
-import sys
-
-
-def get_keywords():
-    """Get the keywords needed to look up the version information."""
-    # these strings will be replaced by git during git-archive.
-    # setup.py/versioneer.py will grep for the variable names, so they must
-    # each be defined on a line of their own. _version.py will just call
-    # get_keywords().
-    git_refnames = "$Format:%d$"
-    git_full = "$Format:%H$"
-    git_date = "$Format:%ci$"
-    keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
-    return keywords
-
-
-class VersioneerConfig:
-    """Container for Versioneer configuration parameters."""
-
-
-def get_config():
-    """Create, populate and return the VersioneerConfig() object."""
-    # these strings are filled in when 'setup.py versioneer' creates
-    # _version.py
-    cfg = VersioneerConfig()
-    cfg.VCS = "git"
-    cfg.style = "pep440"
-    cfg.tag_prefix = ""
-    cfg.parentdir_prefix = "datasette-"
-    cfg.versionfile_source = "datasette/_version.py"
-    cfg.verbose = False
-    return cfg
-
-
-class NotThisMethod(Exception):
-    """Exception raised if a method is not valid for the current scenario."""
-
-
-LONG_VERSION_PY = {}
-HANDLERS = {}
-
-
-def register_vcs_handler(vcs, method):  # decorator
-    """Decorator to mark a method as the handler for a particular VCS."""
-
-    def decorate(f):
-        """Store f in HANDLERS[vcs][method]."""
-        if vcs not in HANDLERS:
-            HANDLERS[vcs] = {}
-        HANDLERS[vcs][method] = f
-        return f
-
-    return decorate
-
-
-def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None):
-    """Call the given command(s)."""
-    assert isinstance(commands, list)
-    p = None
-    for c in commands:
-        try:
-            dispcmd = str([c] + args)
-            # remember shell=False, so use git.cmd on windows, not just git
-            p = subprocess.Popen(
-                [c] + args,
-                cwd=cwd,
-                env=env,
-                stdout=subprocess.PIPE,
-                stderr=(subprocess.PIPE if hide_stderr else None),
-            )
-            break
-        except EnvironmentError:
-            e = sys.exc_info()[1]
-            if e.errno == errno.ENOENT:
-                continue
-            if verbose:
-                print("unable to run %s" % dispcmd)
-                print(e)
-            return None, None
-    else:
-        if verbose:
-            print("unable to find command, tried %s" % (commands,))
-        return None, None
-    stdout = p.communicate()[0].strip()
-    if sys.version_info[0] >= 3:
-        stdout = stdout.decode()
-    if p.returncode != 0:
-        if verbose:
-            print("unable to run %s (error)" % dispcmd)
-            print("stdout was %s" % stdout)
-        return None, p.returncode
-    return stdout, p.returncode
-
-
-def versions_from_parentdir(parentdir_prefix, root, verbose):
-    """Try to determine the version from the parent directory name.
-
-    Source tarballs conventionally unpack into a directory that includes both
-    the project name and a version string. We will also support searching up
-    two directory levels for an appropriately named parent directory
-    """
-    rootdirs = []
-
-    for i in range(3):
-        dirname = os.path.basename(root)
-        if dirname.startswith(parentdir_prefix):
-            return {
-                "version": dirname[len(parentdir_prefix) :],
-                "full-revisionid": None,
-                "dirty": False,
-                "error": None,
-                "date": None,
-            }
-        else:
-            rootdirs.append(root)
-            root = os.path.dirname(root)  # up a level
-
-    if verbose:
-        print(
-            "Tried directories %s but none started with prefix %s"
-            % (str(rootdirs), parentdir_prefix)
-        )
-    raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
-
-
-@register_vcs_handler("git", "get_keywords")
-def git_get_keywords(versionfile_abs):
-    """Extract version information from the given file."""
-    # the code embedded in _version.py can just fetch the value of these
-    # keywords. When used from setup.py, we don't want to import _version.py,
-    # so we do it with a regexp instead. This function is not used from
-    # _version.py.
-    keywords = {}
-    try:
-        f = open(versionfile_abs, "r")
-        for line in f.readlines():
-            if line.strip().startswith("git_refnames ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["refnames"] = mo.group(1)
-            if line.strip().startswith("git_full ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["full"] = mo.group(1)
-            if line.strip().startswith("git_date ="):
-                mo = re.search(r'=\s*"(.*)"', line)
-                if mo:
-                    keywords["date"] = mo.group(1)
-        f.close()
-    except EnvironmentError:
-        pass
-    return keywords
-
-
-@register_vcs_handler("git", "keywords")
-def git_versions_from_keywords(keywords, tag_prefix, verbose):
-    """Get version information from git keywords."""
-    if not keywords:
-        raise NotThisMethod("no keywords at all, weird")
-    date = keywords.get("date")
-    if date is not None:
-        # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
-        # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
-        # -like" string, which we must then edit to make compliant), because
-        # it's been around since git-1.5.3, and it's too difficult to
-        # discover which version we're using, or to work around using an
-        # older one.
-        date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
-    refnames = keywords["refnames"].strip()
-    if refnames.startswith("$Format"):
-        if verbose:
-            print("keywords are unexpanded, not using")
-        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
-    refs = set([r.strip() for r in refnames.strip("()").split(",")])
-    # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
-    # just "foo-1.0". If we see a "tag: " prefix, prefer those.
-    TAG = "tag: "
-    tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)])
-    if not tags:
-        # Either we're using git < 1.8.3, or there really are no tags. We use
-        # a heuristic: assume all version tags have a digit. The old git %d
-        # expansion behaves like git log --decorate=short and strips out the
-        # refs/heads/ and refs/tags/ prefixes that would let us distinguish
-        # between branches and tags. By ignoring refnames without digits, we
-        # filter out many common branch names like "release" and
-        # "stabilization", as well as "HEAD" and "master".
-        tags = set([r for r in refs if re.search(r"\d", r)])
-        if verbose:
-            print("discarding '%s', no digits" % ",".join(refs - tags))
-    if verbose:
-        print("likely tags: %s" % ",".join(sorted(tags)))
-    for ref in sorted(tags):
-        # sorting will prefer e.g. "2.0" over "2.0rc1"
-        if ref.startswith(tag_prefix):
-            r = ref[len(tag_prefix) :]
-            if verbose:
-                print("picking %s" % r)
-            return {
-                "version": r,
-                "full-revisionid": keywords["full"].strip(),
-                "dirty": False,
-                "error": None,
-                "date": date,
-            }
-    # no suitable tags, so version is "0+unknown", but full hex is still there
-    if verbose:
-        print("no suitable tags, using unknown + full revision id")
-    return {
-        "version": "0+unknown",
-        "full-revisionid": keywords["full"].strip(),
-        "dirty": False,
-        "error": "no suitable tags",
-        "date": None,
-    }
-
-
-@register_vcs_handler("git", "pieces_from_vcs")
-def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
-    """Get version from 'git describe' in the root of the source tree.
-
-    This only gets called if the git-archive 'subst' keywords were *not*
-    expanded, and _version.py hasn't already been rewritten with a short
-    version string, meaning we're inside a checked out source tree.
-    """
-    GITS = ["git"]
-    if sys.platform == "win32":
-        GITS = ["git.cmd", "git.exe"]
-
-    out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True)
-    if rc != 0:
-        if verbose:
-            print("Directory %s not under git control" % root)
-        raise NotThisMethod("'git rev-parse --git-dir' returned error")
-
-    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
-    # if there isn't one, this yields HEX[-dirty] (no NUM)
-    describe_out, rc = run_command(
-        GITS,
-        [
-            "describe",
-            "--tags",
-            "--dirty",
-            "--always",
-            "--long",
-            "--match",
-            "%s*" % tag_prefix,
-        ],
-        cwd=root,
-    )
-    # --long was added in git-1.5.5
-    if describe_out is None:
-        raise NotThisMethod("'git describe' failed")
-    describe_out = describe_out.strip()
-    full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root)
-    if full_out is None:
-        raise NotThisMethod("'git rev-parse' failed")
-    full_out = full_out.strip()
-
-    pieces = {}
-    pieces["long"] = full_out
-    pieces["short"] = full_out[:7]  # maybe improved later
-    pieces["error"] = None
-
-    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
-    # TAG might have hyphens.
-    git_describe = describe_out
-
-    # look for -dirty suffix
-    dirty = git_describe.endswith("-dirty")
-    pieces["dirty"] = dirty
-    if dirty:
-        git_describe = git_describe[: git_describe.rindex("-dirty")]
-
-    # now we have TAG-NUM-gHEX or HEX
-
-    if "-" in git_describe:
-        # TAG-NUM-gHEX
-        mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe)
-        if not mo:
-            # unparseable. Maybe git-describe is misbehaving?
-            pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out
-            return pieces
-
-        # tag
-        full_tag = mo.group(1)
-        if not full_tag.startswith(tag_prefix):
-            if verbose:
-                fmt = "tag '%s' doesn't start with prefix '%s'"
-                print(fmt % (full_tag, tag_prefix))
-            pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (
-                full_tag,
-                tag_prefix,
-            )
-            return pieces
-        pieces["closest-tag"] = full_tag[len(tag_prefix) :]
-
-        # distance: number of commits since tag
-        pieces["distance"] = int(mo.group(2))
-
-        # commit: short hex revision ID
-        pieces["short"] = mo.group(3)
-
-    else:
-        # HEX: no tags
-        pieces["closest-tag"] = None
-        count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root)
-        pieces["distance"] = int(count_out)  # total number of commits
-
-    # commit date: see ISO-8601 comment in git_versions_from_keywords()
-    date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[
-        0
-    ].strip()
-    pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
-
-    return pieces
-
-
-def plus_or_dot(pieces):
-    """Return a + if we don't already have one, else return a ."""
-    if "+" in pieces.get("closest-tag", ""):
-        return "."
-    return "+"
-
-
-def render_pep440(pieces):
-    """Build up version string, with post-release "local version identifier".
-
-    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
-    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
-
-    Exceptions:
-    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += plus_or_dot(pieces)
-            rendered += "%d.g%s" % (pieces["distance"], pieces["short"])
-            if pieces["dirty"]:
-                rendered += ".dirty"
-    else:
-        # exception #1
-        rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"])
-        if pieces["dirty"]:
-            rendered += ".dirty"
-    return rendered
-
-
-def render_pep440_pre(pieces):
-    """TAG[.post.devDISTANCE] -- No -dirty.
-
-    Exceptions:
-    1: no tags. 0.post.devDISTANCE
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"]:
-            rendered += ".post.dev%d" % pieces["distance"]
-    else:
-        # exception #1
-        rendered = "0.post.dev%d" % pieces["distance"]
-    return rendered
-
-
-def render_pep440_post(pieces):
-    """TAG[.postDISTANCE[.dev0]+gHEX] .
-
-    The ".dev0" means dirty. Note that .dev0 sorts backwards
-    (a dirty tree will appear "older" than the corresponding clean one),
-    but you shouldn't be releasing software with -dirty anyways.
-
-    Exceptions:
-    1: no tags. 0.postDISTANCE[.dev0]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += ".post%d" % pieces["distance"]
-            if pieces["dirty"]:
-                rendered += ".dev0"
-            rendered += plus_or_dot(pieces)
-            rendered += "g%s" % pieces["short"]
-    else:
-        # exception #1
-        rendered = "0.post%d" % pieces["distance"]
-        if pieces["dirty"]:
-            rendered += ".dev0"
-        rendered += "+g%s" % pieces["short"]
-    return rendered
-
-
-def render_pep440_old(pieces):
-    """TAG[.postDISTANCE[.dev0]] .
-
-    The ".dev0" means dirty.
-
-    Exceptions:
-    1: no tags. 0.postDISTANCE[.dev0]
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"] or pieces["dirty"]:
-            rendered += ".post%d" % pieces["distance"]
-            if pieces["dirty"]:
-                rendered += ".dev0"
-    else:
-        # exception #1
-        rendered = "0.post%d" % pieces["distance"]
-        if pieces["dirty"]:
-            rendered += ".dev0"
-    return rendered
-
-
-def render_git_describe(pieces):
-    """TAG[-DISTANCE-gHEX][-dirty].
-
-    Like 'git describe --tags --dirty --always'.
-
-    Exceptions:
-    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        if pieces["distance"]:
-            rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
-    else:
-        # exception #1
-        rendered = pieces["short"]
-    if pieces["dirty"]:
-        rendered += "-dirty"
-    return rendered
-
-
-def render_git_describe_long(pieces):
-    """TAG-DISTANCE-gHEX[-dirty].
-
-    Like 'git describe --tags --dirty --always -long'.
-    The distance/hash is unconditional.
-
-    Exceptions:
-    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
-    """
-    if pieces["closest-tag"]:
-        rendered = pieces["closest-tag"]
-        rendered += "-%d-g%s" % (pieces["distance"], pieces["short"])
-    else:
-        # exception #1
-        rendered = pieces["short"]
-    if pieces["dirty"]:
-        rendered += "-dirty"
-    return rendered
-
-
-def render(pieces, style):
-    """Render the given version pieces into the requested style."""
-    if pieces["error"]:
-        return {
-            "version": "unknown",
-            "full-revisionid": pieces.get("long"),
-            "dirty": None,
-            "error": pieces["error"],
-            "date": None,
-        }
-
-    if not style or style == "default":
-        style = "pep440"  # the default
-
-    if style == "pep440":
-        rendered = render_pep440(pieces)
-    elif style == "pep440-pre":
-        rendered = render_pep440_pre(pieces)
-    elif style == "pep440-post":
-        rendered = render_pep440_post(pieces)
-    elif style == "pep440-old":
-        rendered = render_pep440_old(pieces)
-    elif style == "git-describe":
-        rendered = render_git_describe(pieces)
-    elif style == "git-describe-long":
-        rendered = render_git_describe_long(pieces)
-    else:
-        raise ValueError("unknown style '%s'" % style)
-
-    return {
-        "version": rendered,
-        "full-revisionid": pieces["long"],
-        "dirty": pieces["dirty"],
-        "error": None,
-        "date": pieces.get("date"),
-    }
-
-
-def get_versions():
-    """Get version information or return default if unable to do so."""
-    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
-    # __file__, we can work backwards from there to the root. Some
-    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
-    # case we can only use expanded keywords.
-
-    cfg = get_config()
-    verbose = cfg.verbose
-
-    try:
-        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose)
-    except NotThisMethod:
-        pass
-
-    try:
-        root = os.path.realpath(__file__)
-        # versionfile_source is the relative path from the top of the source
-        # tree (where the .git directory might live) to this file. Invert
-        # this to find the root from __file__.
-        for i in cfg.versionfile_source.split("/"):
-            root = os.path.dirname(root)
-    except NameError:
-        return {
-            "version": "0+unknown",
-            "full-revisionid": None,
-            "dirty": None,
-            "error": "unable to find root of source tree",
-            "date": None,
-        }
-
-    try:
-        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
-        return render(pieces, cfg.style)
-    except NotThisMethod:
-        pass
-
-    try:
-        if cfg.parentdir_prefix:
-            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
-    except NotThisMethod:
-        pass
-
-    return {
-        "version": "0+unknown",
-        "full-revisionid": None,
-        "dirty": None,
-        "error": "unable to compute version",
-        "date": None,
-    }

datasette/actor_auth_cookie.py

@@ -0,0 +1,23 @@
+from datasette import hookimpl
+from itsdangerous import BadSignature
+from datasette.utils import baseconv
+import time
+
+
+@hookimpl
+def actor_from_request(datasette, request):
+    if "ds_actor" not in request.cookies:
+        return None
+    try:
+        decoded = datasette.unsign(request.cookies["ds_actor"], "actor")
+        # If it has "e" and "a" keys process the "e" expiry
+        if not isinstance(decoded, dict) or "a" not in decoded:
+            return None
+        expires_at = decoded.get("e")
+        if expires_at:
+            timestamp = int(baseconv.base62.decode(expires_at))
+            if time.time() > timestamp:
+                return None
+        return decoded["a"]
+    except BadSignature:
+        return None

datasette/blob_renderer.py

@@ -0,0 +1,61 @@
+from datasette import hookimpl
+from datasette.utils.asgi import Response, BadRequest
+from datasette.utils import to_css_class
+import hashlib
+
+_BLOB_COLUMN = "_blob_column"
+_BLOB_HASH = "_blob_hash"
+
+
+async def render_blob(datasette, database, rows, columns, request, table, view_name):
+    if _BLOB_COLUMN not in request.args:
+        raise BadRequest(f"?{_BLOB_COLUMN}= is required")
+    blob_column = request.args[_BLOB_COLUMN]
+    if blob_column not in columns:
+        raise BadRequest(f"{blob_column} is not a valid column")
+
+    # If ?_blob_hash= provided, use that to select the row - otherwise use first row
+    blob_hash = None
+    if _BLOB_HASH in request.args:
+        blob_hash = request.args[_BLOB_HASH]
+        for row in rows:
+            value = row[blob_column]
+            if hashlib.sha256(value).hexdigest() == blob_hash:
+                break
+        else:
+            # Loop did not break
+            raise BadRequest(
+                "Link has expired - the requested binary content has changed or could not be found."
+            )
+    else:
+        row = rows[0]
+
+    value = row[blob_column]
+    filename_bits = []
+    if table:
+        filename_bits.append(to_css_class(table))
+    if "pks" in request.url_vars:
+        filename_bits.append(request.url_vars["pks"])
+    filename_bits.append(to_css_class(blob_column))
+    if blob_hash:
+        filename_bits.append(blob_hash[:6])
+    filename = "-".join(filename_bits) + ".blob"
+    headers = {
+        "X-Content-Type-Options": "nosniff",
+        "Content-Disposition": f'attachment; filename="{filename}"',
+    }
+    return Response(
+        body=value or b"",
+        status=200,
+        headers=headers,
+        content_type="application/binary",
+    )
+
+
+@hookimpl
+def register_output_renderer():
+    return {
+        "extension": "blob",
+        "render": render_blob,
+        "can_render": lambda: False,
+    }

datasette/cli.py

@@ -2,120 +2,175 @@ import asyncio
 import uvicorn
 import click
 from click import formatting
+from click.types import CompositeParamType
 from click_default_group import DefaultGroup
+import functools
 import json
 import os
+import pathlib
+from runpy import run_module
 import shutil
 from subprocess import call
 import sys
-from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS, pm
+import textwrap
+import webbrowser
+from .app import (
+    Datasette,
+    DEFAULT_SETTINGS,
+    SETTINGS,
+    SQLITE_LIMIT_ATTACHED,
+    pm,
+)
+from .inspect import inspect_tables
 from .utils import (
+    LoadExtension,
+    StartupError,
     check_connection,
+    deep_dict_update,
+    find_spatialite,
+    parse_metadata,
     ConnectionProblem,
     SpatialiteConnectionProblem,
+    initial_path_for_datasette,
+    pairs_to_nested_config,
     temporary_docker_directory,
     value_as_boolean,
+    SpatialiteNotFound,
     StaticMount,
     ValueAsBooleanError,
 )
+from .utils.sqlite import sqlite3
+from .utils.testing import TestClient
+from .version import __version__
 
 
-class Config(click.ParamType):
-    name = "config"
+def run_sync(coro_func):
+    """Run an async callable to completion on a fresh event loop."""
+    loop = asyncio.new_event_loop()
+    try:
+        asyncio.set_event_loop(loop)
+        return loop.run_until_complete(coro_func())
+    finally:
+        asyncio.set_event_loop(None)
+        loop.close()
+
+
+# Use Rich for tracebacks if it is installed
+try:
+    from rich.traceback import install
+
+    install(show_locals=True)
+except ImportError:
+    pass
+
+
+class Setting(CompositeParamType):
+    name = "setting"
+    arity = 2
 
     def convert(self, config, param, ctx):
-        if ":" not in config:
-            self.fail('"{}" should be name:value'.format(config), param, ctx)
-            return
-        name, value = config.split(":", 1)
-        if name not in DEFAULT_CONFIG:
-            self.fail(
-                "{} is not a valid option (--help-config to see all)".format(name),
-                param,
-                ctx,
-            )
-            return
-        # Type checking
-        default = DEFAULT_CONFIG[name]
-        if isinstance(default, bool):
-            try:
-                return name, value_as_boolean(value)
-            except ValueAsBooleanError:
-                self.fail(
-                    '"{}" should be on/off/true/false/1/0'.format(name), param, ctx
-                )
-                return
-        elif isinstance(default, int):
-            if not value.isdigit():
-                self.fail('"{}" should be an integer'.format(name), param, ctx)
-                return
-            return name, int(value)
-        elif isinstance(default, str):
-            return name, value
-        else:
-            # Should never happen:
-            self.fail("Invalid option")
+        name, value = config
+        if name in DEFAULT_SETTINGS:
+            # For backwards compatibility with how this worked prior to
+            # Datasette 1.0, we turn bare setting names into setting.name
+            # Type checking for those older settings
+            default = DEFAULT_SETTINGS[name]
+            name = "settings.{}".format(name)
+            if isinstance(default, bool):
+                try:
+                    return name, "true" if value_as_boolean(value) else "false"
+                except ValueAsBooleanError:
+                    self.fail(f'"{name}" should be on/off/true/false/1/0', param, ctx)
+            elif isinstance(default, int):
+                if not value.isdigit():
+                    self.fail(f'"{name}" should be an integer', param, ctx)
+                return name, value
+            elif isinstance(default, str):
+                return name, value
+            else:
+                # Should never happen:
+                self.fail("Invalid option")
+        return name, value
+
+
+def sqlite_extensions(fn):
+    fn = click.option(
+        "sqlite_extensions",
+        "--load-extension",
+        type=LoadExtension(),
+        envvar="DATASETTE_LOAD_EXTENSION",
+        multiple=True,
+        help="Path to a SQLite extension to load, and optional entrypoint",
+    )(fn)
+
+    # Wrap it in a custom error handler
+    @functools.wraps(fn)
+    def wrapped(*args, **kwargs):
+        try:
+            return fn(*args, **kwargs)
+        except AttributeError as e:
+            if "enable_load_extension" in str(e):
+                raise click.ClickException(textwrap.dedent("""
+                    Your Python installation does not have the ability to load SQLite extensions.
+
+                    More information: https://datasette.io/help/extensions
+                    """).strip())
+            raise
+
+    return wrapped
 
 
 @click.group(cls=DefaultGroup, default="serve", default_if_no_args=True)
-@click.version_option()
+@click.version_option(version=__version__)
 def cli():
     """
-    Datasette!
+    Datasette is an open source multi-tool for exploring and publishing data
+
+    \b
+    About Datasette: https://datasette.io/
+    Full documentation: https://docs.datasette.io/
     """
 
 
 @cli.command()
 @click.argument("files", type=click.Path(exists=True), nargs=-1)
 @click.option("--inspect-file", default="-")
-@click.option(
-    "sqlite_extensions",
-    "--load-extension",
-    envvar="SQLITE_EXTENSIONS",
-    multiple=True,
-    type=click.Path(exists=True, resolve_path=True),
-    help="Path to a SQLite extension to load",
-)
+@sqlite_extensions
 def inspect(files, inspect_file, sqlite_extensions):
-    app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
+    """
+    Generate JSON summary of provided database files
+
+    This can then be passed to "datasette --inspect-file" to speed up count
+    operations against immutable database files.
+    """
+    inspect_data = run_sync(lambda: inspect_(files, sqlite_extensions))
     if inspect_file == "-":
-        out = sys.stdout
+        sys.stdout.write(json.dumps(inspect_data, indent=2))
     else:
-        out = open(inspect_file, "w")
-    loop = asyncio.get_event_loop()
-    inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions))
-    out.write(json.dumps(inspect_data, indent=2))
+        with open(inspect_file, "w") as fp:
+            fp.write(json.dumps(inspect_data, indent=2))
 
 
 async def inspect_(files, sqlite_extensions):
     app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions)
     data = {}
     for name, database in app.databases.items():
-        counts = await database.table_counts(limit=3600 * 1000)
+        tables = await database.execute_fn(lambda conn: inspect_tables(conn, {}))
         data[name] = {
             "hash": database.hash,
             "size": database.size,
             "file": database.path,
             "tables": {
-                table_name: {"count": table_count}
-                for table_name, table_count in counts.items()
+                table_name: {"count": table["count"]}
+                for table_name, table in tables.items()
             },
         }
     return data
 
 
-class PublishAliases(click.Group):
-    aliases = {"now": "nowv1"}
-
-    def get_command(self, ctx, cmd_name):
-        if cmd_name in self.aliases:
-            return click.Group.get_command(self, ctx, self.aliases[cmd_name])
-        return click.Group.get_command(self, ctx, cmd_name)
-
-
-@cli.group(cls=PublishAliases)
+@cli.group()
 def publish():
-    "Publish specified SQLite database files to the internet along with a Datasette-powered interface and API"
+    """Publish specified SQLite database files to the internet along with a Datasette-powered interface and API"""
     pass
 
 
@@ -125,15 +180,23 @@ pm.hook.publish_subcommand(publish=publish)
 
 @cli.command()
 @click.option("--all", help="Include built-in default plugins", is_flag=True)
+@click.option(
+    "--requirements", help="Output requirements.txt of installed plugins", is_flag=True
+)
 @click.option(
     "--plugins-dir",
     type=click.Path(exists=True, file_okay=False, dir_okay=True),
     help="Path to directory containing custom plugins",
 )
-def plugins(all, plugins_dir):
-    "List currently available plugins"
+def plugins(all, requirements, plugins_dir):
+    """List currently installed plugins"""
     app = Datasette([], plugins_dir=plugins_dir)
-    click.echo(json.dumps(app.plugins(all), indent=4))
+    if requirements:
+        for plugin in app._plugins():
+            if plugin["version"]:
+                click.echo("{}=={}".format(plugin["name"], plugin["version"]))
+    else:
+        click.echo(json.dumps(app._plugins(all=all), indent=4))
 
 
 @cli.command()
@@ -147,10 +210,10 @@ def plugins(all, plugins_dir):
     "-m",
     "--metadata",
     type=click.File(mode="r"),
-    help="Path to JSON file containing metadata to publish",
+    help="Path to JSON/YAML file containing metadata to publish",
 )
 @click.option("--extra-options", help="Extra options to pass to datasette serve")
-@click.option("--branch", help="Install datasette from a GitHub branch e.g. master")
+@click.option("--branch", help="Install datasette from a GitHub branch e.g. main")
 @click.option(
     "--template-dir",
     type=click.Path(exists=True, file_okay=False, dir_okay=True),
@@ -173,7 +236,17 @@ def plugins(all, plugins_dir):
 @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension")
 @click.option("--version-note", help="Additional note to show on /-/versions")
 @click.option(
-    "-p", "--port", default=8001, help="Port to run the server on, defaults to 8001",
+    "--secret",
+    help="Secret used for signing secure values, such as signed cookies",
+    envvar="DATASETTE_PUBLISH_SECRET",
+    default=lambda: os.urandom(32).hex(),
+)
+@click.option(
+    "-p",
+    "--port",
+    default=8001,
+    type=click.IntRange(1, 65535),
+    help="Port to run the server on, defaults to 8001",
 )
 @click.option("--title", help="Title for metadata")
 @click.option("--license", help="License label for metadata")
@@ -194,10 +267,11 @@ def package(
     install,
     spatialite,
     version_note,
+    secret,
     port,
-    **extra_metadata
+    **extra_metadata,
 ):
-    "Package specified SQLite files into a new datasette Docker container"
+    """Package SQLite files into a Datasette Docker container"""
     if not shutil.which("docker"):
         click.secho(
             ' The package command requires "docker" to be installed and configured ',
@@ -210,16 +284,17 @@ def package(
     with temporary_docker_directory(
         files,
         "datasette",
-        metadata,
-        extra_options,
-        branch,
-        template_dir,
-        plugins_dir,
-        static,
-        install,
-        spatialite,
-        version_note,
-        extra_metadata,
+        metadata=metadata,
+        extra_options=extra_options,
+        branch=branch,
+        template_dir=template_dir,
+        plugins_dir=plugins_dir,
+        static=static,
+        install=install,
+        spatialite=spatialite,
+        version_note=version_note,
+        secret=secret,
+        extra_metadata=extra_metadata,
         port=port,
     ):
         args = ["docker", "build"]
@@ -231,7 +306,48 @@ def package(
 
 
 @cli.command()
-@click.argument("files", type=click.Path(exists=True), nargs=-1)
+@click.argument("packages", nargs=-1)
+@click.option(
+    "-U", "--upgrade", is_flag=True, help="Upgrade packages to latest version"
+)
+@click.option(
+    "-r",
+    "--requirement",
+    type=click.Path(exists=True),
+    help="Install from requirements file",
+)
+@click.option(
+    "-e",
+    "--editable",
+    help="Install a project in editable mode from this path",
+)
+def install(packages, upgrade, requirement, editable):
+    """Install plugins and packages from PyPI into the same environment as Datasette"""
+    if not packages and not requirement and not editable:
+        raise click.UsageError("Please specify at least one package to install")
+    args = ["pip", "install"]
+    if upgrade:
+        args += ["--upgrade"]
+    if editable:
+        args += ["--editable", editable]
+    if requirement:
+        args += ["-r", requirement]
+    args += list(packages)
+    sys.argv = args
+    run_module("pip", run_name="__main__")
+
+
+@cli.command()
+@click.argument("packages", nargs=-1, required=True)
+@click.option("-y", "--yes", is_flag=True, help="Don't ask for confirmation")
+def uninstall(packages, yes):
+    """Uninstall plugins and Python packages from the Datasette environment"""
+    sys.argv = ["pip", "uninstall"] + list(packages) + (["-y"] if yes else [])
+    run_module("pip", run_name="__main__")
+
+
+@cli.command()
+@click.argument("files", type=click.Path(), nargs=-1)
 @click.option(
     "-i",
     "--immutable",
@@ -253,27 +369,22 @@ def package(
     "-p",
     "--port",
     default=8001,
+    type=click.IntRange(0, 65535),
     help="Port for server, defaults to 8001. Use -p 0 to automatically assign an available port.",
 )
 @click.option(
-    "--debug", is_flag=True, help="Enable debug mode - useful for development"
+    "--uds",
+    help="Bind to a Unix domain socket",
 )
 @click.option(
     "--reload",
     is_flag=True,
-    help="Automatically reload if database or code change detected - useful for development",
+    help="Automatically reload if code or metadata change detected - useful for development",
 )
 @click.option(
     "--cors", is_flag=True, help="Enable CORS by serving Access-Control-Allow-Origin: *"
 )
-@click.option(
-    "sqlite_extensions",
-    "--load-extension",
-    envvar="SQLITE_EXTENSIONS",
-    multiple=True,
-    type=click.Path(exists=True, resolve_path=True),
-    help="Path to a SQLite extension to load",
-)
+@sqlite_extensions
 @click.option(
     "--inspect-file", help='Path to JSON file created using "datasette inspect"'
 )
@@ -281,7 +392,7 @@ def package(
     "-m",
     "--metadata",
     type=click.File(mode="r"),
-    help="Path to JSON file containing license/source metadata",
+    help="Path to JSON/YAML file containing license/source metadata",
 )
 @click.option(
     "--template-dir",
@@ -299,21 +410,99 @@ def package(
     help="Serve static files from this directory at /MOUNT/...",
     multiple=True,
 )
-@click.option("--memory", is_flag=True, help="Make :memory: database available")
+@click.option("--memory", is_flag=True, help="Make /_memory database available")
 @click.option(
+    "-c",
     "--config",
-    type=Config(),
-    help="Set config option using configname:value datasette.readthedocs.io/en/latest/config.html",
+    type=click.File(mode="r"),
+    help="Path to JSON/YAML Datasette configuration file",
+)
+@click.option(
+    "-s",
+    "--setting",
+    "settings",
+    type=Setting(),
+    help="nested.key, value setting to use in Datasette configuration",
     multiple=True,
 )
+@click.option(
+    "--secret",
+    help="Secret used for signing secure values, such as signed cookies",
+    envvar="DATASETTE_SECRET",
+)
+@click.option(
+    "--root",
+    help="Output URL that sets a cookie authenticating the root user",
+    is_flag=True,
+)
+@click.option(
+    "--default-deny",
+    help="Deny all permissions by default",
+    is_flag=True,
+)
+@click.option(
+    "--get",
+    help="Run an HTTP GET request against this path, print results and exit",
+)
+@click.option(
+    "--headers",
+    is_flag=True,
+    help="Include HTTP headers in --get output",
+)
+@click.option(
+    "--token",
+    help="API token to send with --get requests",
+)
+@click.option(
+    "--actor",
+    help="Actor to use for --get requests (JSON string)",
+)
 @click.option("--version-note", help="Additional note to show on /-/versions")
-@click.option("--help-config", is_flag=True, help="Show available config options")
+@click.option("--help-settings", is_flag=True, help="Show available settings")
+@click.option("--pdb", is_flag=True, help="Launch debugger on any errors")
+@click.option(
+    "-o",
+    "--open",
+    "open_browser",
+    is_flag=True,
+    help="Open Datasette in your web browser",
+)
+@click.option(
+    "--create",
+    is_flag=True,
+    help="Create database files if they do not exist",
+)
+@click.option(
+    "--crossdb",
+    is_flag=True,
+    help="Enable cross-database joins using the /_memory database",
+)
+@click.option(
+    "--nolock",
+    is_flag=True,
+    help="Ignore locking, open locked files in read-only mode",
+)
+@click.option(
+    "--ssl-keyfile",
+    help="SSL key file",
+    envvar="DATASETTE_SSL_KEYFILE",
+)
+@click.option(
+    "--ssl-certfile",
+    help="SSL certificate file",
+    envvar="DATASETTE_SSL_CERTFILE",
+)
+@click.option(
+    "--internal",
+    type=click.Path(),
+    help="Path to a persistent Datasette internal SQLite database",
+)
 def serve(
     files,
     immutable,
     host,
     port,
-    debug,
+    uds,
     reload,
     cors,
     sqlite_extensions,
@@ -324,17 +513,34 @@ def serve(
     static,
     memory,
     config,
+    settings,
+    secret,
+    root,
+    default_deny,
+    get,
+    headers,
+    token,
+    actor,
     version_note,
-    help_config,
+    help_settings,
+    pdb,
+    open_browser,
+    create,
+    crossdb,
+    nolock,
+    ssl_keyfile,
+    ssl_certfile,
+    internal,
+    return_instance=False,
 ):
     """Serve up specified SQLite database files with a web UI"""
-    if help_config:
+    if help_settings:
         formatter = formatting.HelpFormatter()
-        with formatter.section("Config options"):
+        with formatter.section("Settings"):
             formatter.write_dl(
                 [
-                    (option.name, "{} (default={})".format(option.help, option.default))
-                    for option in CONFIG_OPTIONS
+                    (option.name, f"{option.help} (default={option.default})")
+                    for option in SETTINGS
                 ]
             )
         click.echo(formatter.getvalue())
@@ -342,43 +548,320 @@ def serve(
     if reload:
         import hupper
 
-        reloader = hupper.start_reloader("datasette.cli.serve")
+        reloader = hupper.start_reloader("datasette.cli.cli")
         if immutable:
             reloader.watch_files(immutable)
+        if config:
+            reloader.watch_files([config.name])
         if metadata:
             reloader.watch_files([metadata.name])
 
     inspect_data = None
     if inspect_file:
-        inspect_data = json.load(open(inspect_file))
+        with open(inspect_file) as fp:
+            inspect_data = json.load(fp)
 
     metadata_data = None
     if metadata:
-        metadata_data = json.loads(metadata.read())
+        metadata_data = parse_metadata(metadata.read())
 
-    click.echo(
-        "Serve! files={} (immutables={}) on port {}".format(files, immutable, port)
-    )
-    ds = Datasette(
-        files,
+    config_data = None
+    if config:
+        config_data = parse_metadata(config.read())
+
+    config_data = config_data or {}
+
+    # Merge in settings from -s/--setting
+    if settings:
+        settings_updates = pairs_to_nested_config(settings)
+        # Merge recursively, to avoid over-writing nested values
+        # https://github.com/simonw/datasette/issues/2389
+        deep_dict_update(config_data, settings_updates)
+
+    kwargs = dict(
         immutables=immutable,
-        cache_headers=not debug and not reload,
+        cache_headers=not reload,
         cors=cors,
         inspect_data=inspect_data,
+        config=config_data,
         metadata=metadata_data,
         sqlite_extensions=sqlite_extensions,
         template_dir=template_dir,
         plugins_dir=plugins_dir,
         static_mounts=static,
-        config=dict(config),
+        settings=None,  # These are passed in config= now
         memory=memory,
+        secret=secret,
         version_note=version_note,
+        pdb=pdb,
+        crossdb=crossdb,
+        nolock=nolock,
+        internal=internal,
+        default_deny=default_deny,
     )
-    # Run async sanity checks - but only if we're not under pytest
-    asyncio.get_event_loop().run_until_complete(check_databases(ds))
+
+    # Separate directories from files
+    directories = [f for f in files if os.path.isdir(f)]
+    file_paths = [f for f in files if not os.path.isdir(f)]
+
+    # Handle config_dir - only one directory allowed
+    if len(directories) > 1:
+        raise click.ClickException(
+            "Cannot pass multiple directories. Pass a single directory as config_dir."
+        )
+    elif len(directories) == 1:
+        kwargs["config_dir"] = pathlib.Path(directories[0])
+
+    # Verify list of files, create if needed (and --create)
+    for file in file_paths:
+        if not pathlib.Path(file).exists():
+            if create:
+                conn = sqlite3.connect(file)
+                conn.execute("vacuum")
+                conn.close()
+            else:
+                raise click.ClickException(
+                    "Invalid value for '[FILES]...': Path '{}' does not exist.".format(
+                        file
+                    )
+                )
+
+    # Check for duplicate files by resolving all paths to their absolute forms
+    # Collect all database files that will be loaded (explicit files + config_dir files)
+    all_db_files = []
+
+    # Add explicit files
+    for file in file_paths:
+        all_db_files.append((file, pathlib.Path(file).resolve()))
+
+    # Add config_dir databases if config_dir is set
+    if "config_dir" in kwargs:
+        config_dir = kwargs["config_dir"]
+        for ext in ("db", "sqlite", "sqlite3"):
+            for db_file in config_dir.glob(f"*.{ext}"):
+                all_db_files.append((str(db_file), db_file.resolve()))
+
+    # Check for duplicates
+    seen = {}
+    for original_path, resolved_path in all_db_files:
+        if resolved_path in seen:
+            raise click.ClickException(
+                f"Duplicate database file: '{original_path}' and '{seen[resolved_path]}' "
+                f"both refer to {resolved_path}"
+            )
+        seen[resolved_path] = original_path
+
+    files = file_paths
+
+    try:
+        ds = Datasette(files, **kwargs)
+    except SpatialiteNotFound:
+        raise click.ClickException("Could not find SpatiaLite extension")
+    except StartupError as e:
+        raise click.ClickException(e.args[0])
+
+    if return_instance:
+        # Private utility mechanism for writing unit tests
+        return ds
+
+    # Run async soundness checks before startup hooks, since invoke_startup
+    # now populates internal tables which requires querying each database
+    run_sync(lambda: check_databases(ds))
+
+    # Run the "startup" plugin hooks
+    try:
+        run_sync(ds.invoke_startup)
+    except StartupError as e:
+        raise click.ClickException(e.args[0])
+
+    if headers and not get:
+        raise click.ClickException("--headers can only be used with --get")
+
+    if token and not get:
+        raise click.ClickException("--token can only be used with --get")
+
+    if get:
+        client = TestClient(ds)
+        request_headers = {}
+        if token:
+            request_headers["Authorization"] = "Bearer {}".format(token)
+        cookies = {}
+        if actor:
+            cookies["ds_actor"] = client.actor_cookie(json.loads(actor))
+        response = client.get(get, headers=request_headers, cookies=cookies)
+
+        if headers:
+            # Output HTTP status code, headers, two newlines, then the response body
+            click.echo(f"HTTP/1.1 {response.status}")
+            for key, value in response.headers.items():
+                click.echo(f"{key}: {value}")
+            if response.text:
+                click.echo()
+                click.echo(response.text)
+        else:
+            click.echo(response.text)
+
+        exit_code = 0 if response.status == 200 else 1
+        sys.exit(exit_code)
+        return
 
     # Start the server
-    uvicorn.run(ds.app(), host=host, port=port, log_level="info")
+    url = None
+    if root:
+        ds.root_enabled = True
+        url = "http://{}:{}{}?token={}".format(
+            host, port, ds.urls.path("-/auth-token"), ds._root_token
+        )
+        click.echo(url)
+    if open_browser:
+        if url is None:
+            # Figure out most convenient URL - to table, database or homepage
+            path = run_sync(lambda: initial_path_for_datasette(ds))
+            url = f"http://{host}:{port}{path}"
+        webbrowser.open(url)
+    uvicorn_kwargs = dict(
+        host=host, port=port, log_level="info", lifespan="on", workers=1
+    )
+    if uds:
+        uvicorn_kwargs["uds"] = uds
+    if ssl_keyfile:
+        uvicorn_kwargs["ssl_keyfile"] = ssl_keyfile
+    if ssl_certfile:
+        uvicorn_kwargs["ssl_certfile"] = ssl_certfile
+    uvicorn.run(ds.app(), **uvicorn_kwargs)
+
+
+@cli.command()
+@click.argument("id")
+@click.option(
+    "--secret",
+    help="Secret used for signing the API tokens",
+    envvar="DATASETTE_SECRET",
+    required=True,
+)
+@click.option(
+    "-e",
+    "--expires-after",
+    help="Token should expire after this many seconds",
+    type=int,
+)
+@click.option(
+    "alls",
+    "-a",
+    "--all",
+    type=str,
+    metavar="ACTION",
+    multiple=True,
+    help="Restrict token to this action",
+)
+@click.option(
+    "databases",
+    "-d",
+    "--database",
+    type=(str, str),
+    metavar="DB ACTION",
+    multiple=True,
+    help="Restrict token to this action on this database",
+)
+@click.option(
+    "resources",
+    "-r",
+    "--resource",
+    type=(str, str, str),
+    metavar="DB RESOURCE ACTION",
+    multiple=True,
+    help="Restrict token to this action on this database resource (a table, SQL view or named query)",
+)
+@click.option(
+    "--debug",
+    help="Show decoded token",
+    is_flag=True,
+)
+@click.option(
+    "--plugins-dir",
+    type=click.Path(exists=True, file_okay=False, dir_okay=True),
+    help="Path to directory containing custom plugins",
+)
+def create_token(
+    id, secret, expires_after, alls, databases, resources, debug, plugins_dir
+):
+    """
+    Create a signed API token for the specified actor ID
+
+    Example:
+
+        datasette create-token root --secret mysecret
+
+    To allow only "view-database-download" for all databases:
+
+    \b
+        datasette create-token root --secret mysecret \\
+            --all view-database-download
+
+    To allow "create-table" against a specific database:
+
+    \b
+        datasette create-token root --secret mysecret \\
+            --database mydb create-table
+
+    To allow "insert-row" against a specific table:
+
+    \b
+        datasette create-token root --secret myscret \\
+            --resource mydb mytable insert-row
+
+    Restricted actions can be specified multiple times using
+    multiple --all, --database, and --resource options.
+
+    Add --debug to see a decoded version of the token.
+    """
+    ds = Datasette(secret=secret, plugins_dir=plugins_dir)
+
+    # Run ds.invoke_startup() in an event loop
+    try:
+        run_sync(ds.invoke_startup)
+    except StartupError as e:
+        raise click.ClickException(e.args[0])
+
+    # Warn about any unknown actions
+    actions = []
+    actions.extend(alls)
+    actions.extend([p[1] for p in databases])
+    actions.extend([p[2] for p in resources])
+    for action in actions:
+        if not ds.actions.get(action):
+            click.secho(
+                f"  Unknown permission: {action} ",
+                fg="red",
+                err=True,
+            )
+
+    from datasette.tokens import TokenRestrictions
+
+    restrictions = TokenRestrictions()
+    for action in alls:
+        restrictions.allow_all(action)
+    for database, action in databases:
+        restrictions.allow_database(database, action)
+    for database, resource, action in resources:
+        restrictions.allow_resource(database, resource, action)
+
+    token = run_sync(
+        lambda: ds.create_token(
+            id,
+            expires_after=expires_after,
+            restrictions=restrictions,
+            handler="signed",
+        )
+    )
+    click.echo(token)
+    if debug:
+        encoded = token[len("dstok_") :]
+        click.echo("\nDecoded:\n")
+        click.echo(json.dumps(ds.unsign(encoded, namespace="token"), indent=2))
+
+
+pm.hook.register_commands(cli=cli)
 
 
 async def check_databases(ds):
@@ -386,16 +869,33 @@ async def check_databases(ds):
     # to confirm they are all usable
     for database in list(ds.databases.values()):
         try:
-            await database.execute_against_connection_in_thread(check_connection)
+            await database.execute_fn(check_connection)
         except SpatialiteConnectionProblem:
+            suggestion = ""
+            try:
+                find_spatialite()
+                suggestion = "\n\nTry adding the --load-extension=spatialite option."
+            except SpatialiteNotFound:
+                pass
             raise click.UsageError(
                 "It looks like you're trying to load a SpatiaLite"
-                " database without first loading the SpatiaLite module."
-                "\n\nRead more: https://datasette.readthedocs.io/en/latest/spatialite.html"
+                + " database without first loading the SpatiaLite module."
+                + suggestion
+                + "\n\nRead more: https://docs.datasette.io/en/stable/spatialite.html"
             )
         except ConnectionProblem as e:
             raise click.UsageError(
-                "Connection to {} failed check: {}".format(
-                    database.path, str(e.args[0])
-                )
+                f"Connection to {database.path} failed check: {str(e.args[0])}"
             )
+    # If --crossdb and more than SQLITE_LIMIT_ATTACHED show warning
+    if (
+        ds.crossdb
+        and len([db for db in ds.databases.values() if not db.is_memory])
+        > SQLITE_LIMIT_ATTACHED
+    ):
+        msg = (
+            "Warning: --crossdb only works with the first {} attached databases".format(
+                SQLITE_LIMIT_ATTACHED
+            )
+        )
+        click.echo(click.style(msg, bold=True, fg="yellow"), err=True)

datasette/column_types.py

@@ -0,0 +1,83 @@
+from enum import Enum
+
+
+class SQLiteType(Enum):
+    TEXT = "TEXT"
+    INTEGER = "INTEGER"
+    REAL = "REAL"
+    BLOB = "BLOB"
+    NULL = "NULL"
+
+    @classmethod
+    def from_declared_type(cls, declared_type: str | None) -> "SQLiteType | None":
+        if declared_type is None:
+            return cls.NULL
+
+        normalized = declared_type.strip().upper()
+        if not normalized:
+            return cls.NULL
+
+        if normalized == cls.NULL.value:
+            return cls.NULL
+        if "INT" in normalized:
+            return cls.INTEGER
+        if any(token in normalized for token in ("CHAR", "CLOB", "TEXT")):
+            return cls.TEXT
+        if "BLOB" in normalized:
+            return cls.BLOB
+        if any(
+            token in normalized
+            for token in ("REAL", "FLOA", "DOUB")  # codespell:ignore doub
+        ):
+            return cls.REAL
+
+        return None
+
+
+class ColumnType:
+    """
+    Base class for column types.
+
+    Subclasses must define ``name`` and ``description`` as class attributes:
+
+    - ``name``: Unique identifier string. Lowercase, no spaces.
+      Examples: "markdown", "file", "email", "url", "point", "image".
+    - ``description``: Human-readable label for admin UI dropdowns.
+      Examples: "Markdown text", "File reference", "Email address".
+    - ``sqlite_types``: Optional tuple of SQLiteType values restricting
+      which SQLite column types this ColumnType can be assigned to.
+
+    Instantiate with an optional ``config`` dict to bind per-column
+    configuration::
+
+        ct = MyColumnType(config={"key": "value"})
+        ct.config  # {"key": "value"}
+    """
+
+    name: str
+    description: str
+    sqlite_types: tuple[SQLiteType, ...] | None = None
+
+    def __init__(self, config=None):
+        self.config = config
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        """
+        Return an HTML string to render this cell value, or None to
+        fall through to the default render_cell plugin hook chain.
+        """
+        return None
+
+    async def validate(self, value, datasette):
+        """
+        Validate a value before it is written. Return None if valid,
+        or a string error message if invalid.
+        """
+        return None
+
+    async def transform_value(self, value, datasette):
+        """
+        Transform a value before it appears in JSON API output.
+        Return the transformed value. Default: return unchanged.
+        """
+        return value

datasette/csrf.py

@@ -0,0 +1,178 @@
+"""
+Header-based CSRF (Cross-Origin) protection.
+
+Datasette uses the Sec-Fetch-Site + Origin header approach described in
+Filippo Valsorda's article (https://words.filippo.io/csrf/) and implemented
+in Go 1.25's http.CrossOriginProtection. This replaces the previous
+token-based asgi-csrf mechanism.
+"""
+
+from __future__ import annotations
+
+import secrets
+import urllib.parse
+
+from .utils.asgi import asgi_send
+
+SAFE_METHODS = frozenset({"GET", "HEAD", "OPTIONS"})
+
+DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443}
+
+
+def _normalize_headers(raw_headers):
+    """Lowercase header names; for duplicates, last value wins."""
+    result = {}
+    for name, value in raw_headers:
+        if isinstance(name, str):
+            name = name.encode("latin-1")
+        if isinstance(value, str):
+            value = value.encode("latin-1")
+        result[name.lower()] = value
+    return result
+
+
+def _origin_tuple(value):
+    """
+    Parse an origin-like string into ``(scheme, host, port)`` with default
+    ports filled in. Raises ``ValueError`` for malformed input.
+    """
+    parsed = urllib.parse.urlsplit(value)
+    scheme = (parsed.scheme or "").lower()
+    host = (parsed.hostname or "").lower()
+    if not scheme or not host:
+        raise ValueError("missing scheme or host in {!r}".format(value))
+    port = parsed.port  # may raise ValueError on bad ports
+    if port is None:
+        port = DEFAULT_PORTS.get(scheme)
+    if port is None:
+        raise ValueError("unknown default port for scheme {!r}".format(scheme))
+    return scheme, host, port
+
+
+def _install_legacy_csrftoken(scope):
+    """
+    Populate ``scope["csrftoken"]`` with a callable returning a per-request
+    random token. Provided for plugin compatibility only - core no longer
+    uses this value for CSRF enforcement.
+    """
+
+    def csrftoken():
+        if "_datasette_legacy_csrftoken" not in scope:
+            scope["_datasette_legacy_csrftoken"] = secrets.token_urlsafe(32)
+        return scope["_datasette_legacy_csrftoken"]
+
+    scope["csrftoken"] = csrftoken
+
+
+class CrossOriginProtectionMiddleware:
+    """
+    Modern CSRF protection using the Sec-Fetch-Site and Origin headers.
+
+    Based on Filippo Valsorda's algorithm, as implemented in Go 1.25's
+    http.CrossOriginProtection. See https://words.filippo.io/csrf/
+
+    Unsafe-method requests are allowed through only if they look same-origin.
+    Non-browser clients (curl, etc.) send neither Sec-Fetch-Site nor Origin
+    and are passed through unchanged - CSRF is a browser-only attack.
+    """
+
+    SAFE_METHODS = SAFE_METHODS
+
+    def __init__(self, app, datasette):
+        self.app = app
+        self.datasette = datasette
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        _install_legacy_csrftoken(scope)
+
+        if scope.get("method", "GET") in self.SAFE_METHODS:
+            await self.app(scope, receive, send)
+            return
+
+        headers = _normalize_headers(scope.get("headers") or [])
+
+        authorization = headers.get(b"authorization", b"").decode("latin-1")
+        cookie_header = headers.get(b"cookie")
+        # Bearer-token requests are not ambient browser credentials, so they
+        # are not CSRF-vulnerable. Narrowly exempt them from the header check
+        # before evaluating Sec-Fetch-Site / Origin. Only "Bearer" is exempt;
+        # schemes like Basic or Digest can be browser-managed and ambient.
+        # If the request also carries a Cookie header, ambient cookie auth
+        # could be in play, so do NOT treat it as exempt.
+        if authorization and not cookie_header:
+            parts = authorization.split(None, 1)
+            if parts and parts[0].lower() == "bearer":
+                await self.app(scope, receive, send)
+                return
+
+        origin_bytes = headers.get(b"origin")
+        sec_fetch_site_bytes = headers.get(b"sec-fetch-site")
+        host_bytes = headers.get(b"host", b"")
+        origin = origin_bytes.decode("latin-1") if origin_bytes else None
+        sec_fetch_site = (
+            sec_fetch_site_bytes.decode("latin-1") if sec_fetch_site_bytes else None
+        )
+        host = host_bytes.decode("latin-1")
+
+        # Primary defense: Sec-Fetch-Site (set by browsers, unforgeable from JS)
+        if sec_fetch_site is not None:
+            if sec_fetch_site in ("same-origin", "none"):
+                await self.app(scope, receive, send)
+                return
+            await self._forbid(
+                send,
+                "Sec-Fetch-Site was {!r}, expected 'same-origin' or 'none'".format(
+                    sec_fetch_site
+                ),
+            )
+            return
+
+        # No Sec-Fetch-Site and no Origin -> non-browser client (curl, API, etc.)
+        if origin is None:
+            await self.app(scope, receive, send)
+            return
+
+        # Fallback for older browsers: Origin must match the request's own
+        # scheme + host + port. Compare full origin tuples, not host alone.
+        request_scheme = self._request_scheme(scope)
+        try:
+            origin_tuple = _origin_tuple(origin)
+            expected_tuple = _origin_tuple("{}://{}".format(request_scheme, host))
+        except ValueError:
+            await self._forbid(
+                send,
+                "Malformed Origin {!r} or Host {!r}".format(origin, host),
+            )
+            return
+
+        if origin_tuple == expected_tuple:
+            await self.app(scope, receive, send)
+            return
+
+        await self._forbid(
+            send,
+            "Origin {!r} does not match Host {!r}".format(origin, host),
+        )
+
+    def _request_scheme(self, scope):
+        if self.datasette is not None:
+            try:
+                if self.datasette.setting("force_https_urls"):
+                    return "https"
+            except Exception:
+                pass
+        return scope.get("scheme") or "http"
+
+    async def _forbid(self, send, reason):
+        await asgi_send(
+            send,
+            content=await self.datasette.render_template(
+                "csrf_error.html", {"reason": reason}
+            ),
+            status=403,
+            content_type="text/html; charset=utf-8",
+        )

datasette/database.py

@@ -1,118 +1,490 @@
 import asyncio
-import contextlib
+import atexit
+from collections import namedtuple
+import inspect
+import os
 from pathlib import Path
-import janus
 import queue
+import sqlite_utils
+import sys
+import tempfile
 import threading
 import uuid
 
 from .tracer import trace
 from .utils import (
-    QueryInterrupted,
-    Results,
+    call_with_supported_arguments,
     detect_fts,
     detect_primary_keys,
     detect_spatialite,
     get_all_foreign_keys,
     get_outbound_foreign_keys,
+    md5_not_usedforsecurity,
     sqlite_timelimit,
     sqlite3,
     table_columns,
+    table_column_details,
 )
+from .utils.sql_analysis import SQLAnalysis, analyze_sql_tables
+from .utils.sqlite import sqlite_hidden_table_names
 from .inspect import inspect_hash
 
 connections = threading.local()
 
+AttachedDatabase = namedtuple("AttachedDatabase", ("seq", "name", "file"))
+
+
+class DatasetteClosedError(RuntimeError):
+    """Raised when using a Datasette or Database instance after close()."""
+
+
+_SHUTDOWN = object()
+
 
 class Database:
-    def __init__(self, ds, path=None, is_mutable=False, is_memory=False):
+    # For table counts stop at this many rows:
+    count_limit = 10000
+    _thread_local_id_counter = 1
+
+    def __init__(
+        self,
+        ds,
+        path=None,
+        is_mutable=True,
+        is_memory=False,
+        memory_name=None,
+        mode=None,
+        is_temp_disk=False,
+    ):
+        self.name = None
+        self._thread_local_id = f"x{self._thread_local_id_counter}"
+        Database._thread_local_id_counter += 1
+        self.route = None
         self.ds = ds
         self.path = path
         self.is_mutable = is_mutable
         self.is_memory = is_memory
-        self.hash = None
+        self.memory_name = memory_name
+        self.is_temp_disk = is_temp_disk
+        if memory_name is not None:
+            self.is_memory = True
+        if is_temp_disk:
+            fd, temp_path = tempfile.mkstemp(suffix=".db", prefix="datasette_temp_")
+            os.close(fd)
+            self.path = temp_path
+            self.is_mutable = True
+            self.mode = "rwc"
+            self._wal_enabled = False
+            atexit.register(self._cleanup_temp_file)
+        else:
+            self._wal_enabled = False
+        self.cached_hash = None
         self.cached_size = None
-        self.cached_table_counts = None
+        self._cached_table_counts = None
         self._write_thread = None
         self._write_queue = None
-        if not self.is_mutable:
-            p = Path(path)
-            self.hash = inspect_hash(p)
-            self.cached_size = p.stat().st_size
-            # Maybe use self.ds.inspect_data to populate cached_table_counts
-            if self.ds.inspect_data and self.ds.inspect_data.get(self.name):
-                self.cached_table_counts = {
-                    key: value["count"]
-                    for key, value in self.ds.inspect_data[self.name]["tables"].items()
-                }
+        self._closed = False
+        self._pending_execute_futures = set()
+        self._pending_execute_futures_lock = threading.Lock()
+        # These are used when in non-threaded mode:
+        self._read_connection = None
+        self._write_connection = None
+        # This is used to track all file connections so they can be closed
+        self._all_file_connections = []
+        if not is_temp_disk:
+            self.mode = mode
+
+    def _check_not_closed(self):
+        if self._closed:
+            raise DatasetteClosedError(
+                "Database {!r} has been closed".format(self.name)
+            )
+
+    def _remove_pending_execute_future(self, future):
+        with self._pending_execute_futures_lock:
+            self._pending_execute_futures.discard(future)
+
+    @property
+    def cached_table_counts(self):
+        if self._cached_table_counts is not None:
+            return self._cached_table_counts
+        # Maybe use self.ds.inspect_data to populate cached_table_counts
+        if self.ds.inspect_data and self.ds.inspect_data.get(self.name):
+            self._cached_table_counts = {
+                key: value["count"]
+                for key, value in self.ds.inspect_data[self.name]["tables"].items()
+            }
+        return self._cached_table_counts
+
+    @property
+    def color(self):
+        if self.hash:
+            return self.hash[:6]
+        return md5_not_usedforsecurity(self.name)[:6]
+
+    def suggest_name(self):
+        if self.is_temp_disk:
+            return "_temp_disk"
+        if self.path:
+            return Path(self.path).stem
+        elif self.memory_name:
+            return self.memory_name
+        else:
+            return "db"
 
     def connect(self, write=False):
+        extra_kwargs = {}
+        if write:
+            extra_kwargs["isolation_level"] = "IMMEDIATE"
+        if self.memory_name:
+            uri = "file:{}?mode=memory&cache=shared".format(self.memory_name)
+            conn = sqlite3.connect(
+                uri, uri=True, check_same_thread=False, **extra_kwargs
+            )
+            if not write:
+                conn.execute("PRAGMA query_only=1")
+            return conn
         if self.is_memory:
-            return sqlite3.connect(":memory:")
+            return sqlite3.connect(":memory:", uri=True)
+
         # mode=ro or immutable=1?
         if self.is_mutable:
             qs = "?mode=ro"
+            if self.ds.nolock:
+                qs += "&nolock=1"
         else:
             qs = "?immutable=1"
         assert not (write and not self.is_mutable)
         if write:
             qs = ""
-        return sqlite3.connect(
-            "file:{}{}".format(self.path, qs), uri=True, check_same_thread=False
+        if self.mode is not None:
+            qs = f"?mode={self.mode}"
+        conn = sqlite3.connect(
+            f"file:{self.path}{qs}", uri=True, check_same_thread=False, **extra_kwargs
+        )
+        self._all_file_connections.append(conn)
+        if self.is_temp_disk and not self._wal_enabled:
+            conn.execute("PRAGMA journal_mode=WAL")
+            self._wal_enabled = True
+        return conn
+
+    def close(self):
+        """Release all resources held by this database.
+
+        Idempotent. After close() further calls to execute()/execute_fn()/
+        execute_write()/execute_write_fn() raise DatasetteClosedError.
+        """
+        if self._closed:
+            return
+        with self._pending_execute_futures_lock:
+            if self._closed:
+                return
+            self._closed = True
+            pending_execute_futures = tuple(self._pending_execute_futures)
+        # Shut down the write thread, if any, via a sentinel. The thread
+        # drains any writes already queued before the sentinel and then
+        # closes its own write connection and returns.
+        write_thread = self._write_thread
+        if write_thread is not None and self._write_queue is not None:
+            self._write_queue.put(_SHUTDOWN)
+            write_thread.join(timeout=10)
+            if write_thread.is_alive():
+                sys.stderr.write(
+                    "Datasette: write thread for {!r} did not exit within 10s\n".format(
+                        self.name
+                    )
+                )
+                sys.stderr.flush()
+        for future in pending_execute_futures:
+            try:
+                future.result()
+            except Exception:
+                pass
+        # Close anything still tracked in _all_file_connections
+        for connection in self._all_file_connections:
+            try:
+                connection.close()
+            except Exception:
+                pass
+        self._all_file_connections = []
+        # Drop per-thread cached read connections we can reach
+        try:
+            delattr(connections, self._thread_local_id)
+        except AttributeError:
+            pass
+        # Close non-threaded-mode cached connections if still open
+        if self._read_connection is not None:
+            try:
+                self._read_connection.close()
+            except Exception:
+                pass
+            self._read_connection = None
+        if self._write_connection is not None:
+            try:
+                self._write_connection.close()
+            except Exception:
+                pass
+            self._write_connection = None
+        if self.is_temp_disk:
+            self._cleanup_temp_file()
+
+    def _cleanup_temp_file(self):
+        if self.is_temp_disk and self.path:
+            for suffix in ("", "-wal", "-shm"):
+                try:
+                    os.unlink(self.path + suffix)
+                except OSError:
+                    pass
+
+    async def execute_write(self, sql, params=None, block=True, request=None):
+        self._check_not_closed()
+
+        def _inner(conn):
+            return conn.execute(sql, params or [])
+
+        with trace("sql", database=self.name, sql=sql.strip(), params=params):
+            results = await self.execute_write_fn(_inner, block=block, request=request)
+        return results
+
+    async def execute_write_script(self, sql, block=True, request=None):
+        self._check_not_closed()
+
+        def _inner(conn):
+            return conn.executescript(sql)
+
+        with trace("sql", database=self.name, sql=sql.strip(), executescript=True):
+            results = await self.execute_write_fn(
+                _inner, block=block, transaction=False, request=request
+            )
+        return results
+
+    async def execute_write_many(self, sql, params_seq, block=True, request=None):
+        self._check_not_closed()
+
+        def _inner(conn):
+            count = 0
+
+            def count_params(params):
+                nonlocal count
+                for param in params:
+                    count += 1
+                    yield param
+
+            return conn.executemany(sql, count_params(params_seq)), count
+
+        with trace(
+            "sql", database=self.name, sql=sql.strip(), executemany=True
+        ) as kwargs:
+            results, count = await self.execute_write_fn(
+                _inner, block=block, request=request
+            )
+            kwargs["count"] = count
+        return results
+
+    async def execute_isolated_fn(self, fn):
+        self._check_not_closed()
+        # Open a new connection just for the duration of this function
+        # blocking the write queue to avoid any writes occurring during it
+        if self.ds.executor is None:
+            # non-threaded mode
+            isolated_connection = self.connect(write=True)
+            try:
+                result = fn(isolated_connection)
+            finally:
+                isolated_connection.close()
+                try:
+                    self._all_file_connections.remove(isolated_connection)
+                except ValueError:
+                    # Was probably a memory connection
+                    pass
+            return result
+        else:
+            # Threaded mode - send to write thread
+            return await self._send_to_write_thread(fn, isolated_connection=True)
+
+    async def analyze_sql(self, sql, params=None) -> SQLAnalysis:
+        self._check_not_closed()
+
+        return await self.execute_isolated_fn(
+            lambda conn: analyze_sql_tables(conn, sql, params, database_name=self.name)
         )
 
-    async def execute_write(self, sql, params=None, block=False):
-        def _inner(conn):
-            with conn:
-                return conn.execute(sql, params or [])
+    async def execute_write_fn(self, fn, block=True, transaction=True, request=None):
+        self._check_not_closed()
+        pending_events = []
 
-        return await self.execute_write_fn(_inner, block=block)
+        def track_event(event):
+            pending_events.append(event)
 
-    async def execute_write_fn(self, fn, block=False):
-        task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
+        fn = self._wrap_fn_with_hooks(fn, request, transaction, track_event)
+        if self.ds.executor is None:
+            # non-threaded mode
+            if self._write_connection is None:
+                self._write_connection = self.connect(write=True)
+                self.ds._prepare_connection(self._write_connection, self.name)
+            if transaction:
+                with self._write_connection:
+                    result = fn(self._write_connection)
+            else:
+                result = fn(self._write_connection)
+        else:
+            result = await self._send_to_write_thread(
+                fn, block=block, transaction=transaction
+            )
+        if block:
+            for event in pending_events:
+                await self.ds.track_event(event)
+        else:
+            # For non-blocking writes, spawn a background task to
+            # dispatch events after the write thread completes
+            task_id, reply_future = result
+
+            async def _dispatch_events_after_write():
+                try:
+                    await reply_future
+                except Exception:
+                    # if the write failed, don't emit success events
+                    return
+                for event in pending_events:
+                    await self.ds.track_event(event)
+
+            asyncio.ensure_future(_dispatch_events_after_write())
+            result = task_id
+        return result
+
+    def _wrap_fn_with_hooks(self, fn, request, transaction, track_event):
+        from .plugins import pm
+
+        # Wrap fn so it receives track_event if its signature supports it.
+        # Historically fn was called positionally, so any single-parameter
+        # name (conn, connection, db, ...) worked. Preserve that by only
+        # switching to keyword dependency injection when the callback
+        # explicitly opts in by declaring a `track_event` parameter.
+        original_fn = fn
+
+        if "track_event" in inspect.signature(original_fn).parameters:
+
+            def fn_with_track_event(conn):
+                return call_with_supported_arguments(
+                    original_fn, conn=conn, track_event=track_event
+                )
+
+            fn = fn_with_track_event
+
+        wrappers = pm.hook.write_wrapper(
+            datasette=self.ds,
+            database=self.name,
+            request=request,
+            transaction=transaction,
+        )
+        wrappers = [w for w in wrappers if w is not None]
+        if not wrappers:
+            return fn
+        # Build the wrapped fn by nesting context manager generators.
+        # The first wrapper returned by pluggy is outermost.
+        for wrapper_factory in reversed(wrappers):
+            fn = _apply_write_wrapper(fn, wrapper_factory, track_event)
+        return fn
+
+    async def _send_to_write_thread(
+        self, fn, block=True, isolated_connection=False, transaction=True
+    ):
         if self._write_queue is None:
             self._write_queue = queue.Queue()
         if self._write_thread is None:
             self._write_thread = threading.Thread(
                 target=self._execute_writes, daemon=True
             )
+            self._write_thread.name = "_execute_writes for database {}".format(
+                self.name
+            )
             self._write_thread.start()
-        reply_queue = janus.Queue()
-        self._write_queue.put(WriteTask(fn, task_id, reply_queue))
+        task_id = uuid.uuid5(uuid.NAMESPACE_DNS, "datasette.io")
+        loop = asyncio.get_running_loop()
+        reply_future = loop.create_future()
+        self._write_queue.put(
+            WriteTask(fn, task_id, loop, reply_future, isolated_connection, transaction)
+        )
         if block:
-            result = await reply_queue.async_q.get()
-            if isinstance(result, Exception):
-                raise result
-            else:
-                return result
+            return await reply_future
         else:
-            return task_id
+            return task_id, reply_future
 
     def _execute_writes(self):
         # Infinite looping thread that protects the single write connection
         # to this database
-        conn = self.connect(write=True)
+        conn_exception = None
+        conn = None
+        try:
+            conn = self.connect(write=True)
+            self.ds._prepare_connection(conn, self.name)
+        except Exception as e:
+            conn_exception = e
         while True:
             task = self._write_queue.get()
-            try:
-                result = task.fn(conn)
-            except Exception as e:
-                print(e)
-                result = e
-            task.reply_queue.sync_q.put(result)
+            if task is _SHUTDOWN:
+                if conn is not None:
+                    try:
+                        conn.close()
+                    except Exception:
+                        pass
+                return
+            exception = None
+            result = None
+            if conn_exception is not None:
+                exception = conn_exception
+            elif task.isolated_connection:
+                isolated_connection = self.connect(write=True)
+                try:
+                    result = task.fn(isolated_connection)
+                except Exception as e:
+                    sys.stderr.write("{}\n".format(e))
+                    sys.stderr.flush()
+                    exception = e
+                finally:
+                    isolated_connection.close()
+                    try:
+                        self._all_file_connections.remove(isolated_connection)
+                    except ValueError:
+                        # Was probably a memory connection
+                        pass
+            else:
+                try:
+                    if task.transaction:
+                        with conn:
+                            result = task.fn(conn)
+                    else:
+                        result = task.fn(conn)
+                except Exception as e:
+                    sys.stderr.write("{}\n".format(e))
+                    sys.stderr.flush()
+                    exception = e
+            _deliver_write_result(task, result, exception)
 
-    async def execute_against_connection_in_thread(self, fn):
+    async def execute_fn(self, fn):
+        self._check_not_closed()
+        if self.ds.executor is None:
+            # non-threaded mode
+            if self._read_connection is None:
+                self._read_connection = self.connect()
+                self.ds._prepare_connection(self._read_connection, self.name)
+            return fn(self._read_connection)
+
+        # threaded mode
         def in_thread():
-            conn = getattr(connections, self.name, None)
+            conn = getattr(connections, self._thread_local_id, None)
             if not conn:
                 conn = self.connect()
-                self.ds.prepare_connection(conn, self.name)
-                setattr(connections, self.name, conn)
+                self.ds._prepare_connection(conn, self.name)
+                setattr(connections, self._thread_local_id, conn)
             return fn(conn)
 
-        return await asyncio.get_event_loop().run_in_executor(
-            self.ds.executor, in_thread
-        )
+        with self._pending_execute_futures_lock:
+            self._check_not_closed()
+            future = self.ds.executor.submit(in_thread)
+            self._pending_execute_futures.add(future)
+        future.add_done_callback(self._remove_pending_execute_future)
+        return await asyncio.wrap_future(future)
 
     async def execute(
         self,
@@ -124,6 +496,7 @@ class Database:
         log_sql_errors=True,
     ):
         """Executes sql against db_name in a thread"""
+        self._check_not_closed()
         page_size = page_size or self.ds.page_size
 
         def sql_operation_in_thread(conn):
@@ -134,7 +507,7 @@ class Database:
             with sqlite_timelimit(conn, time_limit_ms):
                 try:
                     cursor = conn.cursor()
-                    cursor.execute(sql, params or {})
+                    cursor.execute(sql, params if params is not None else {})
                     max_returned_rows = self.ds.max_returned_rows
                     if max_returned_rows == page_size:
                         max_returned_rows += 1
@@ -149,11 +522,12 @@ class Database:
                     if e.args == ("interrupted",):
                         raise QueryInterrupted(e, sql, params)
                     if log_sql_errors:
-                        print(
-                            "ERROR: conn={}, sql = {}, params = {}: {}".format(
+                        sys.stderr.write(
+                            "ERROR: conn={}, sql = {}, params = {}: {}\n".format(
                                 conn, repr(sql), params, e
                             )
                         )
+                        sys.stderr.flush()
                     raise
 
             if truncate:
@@ -163,19 +537,37 @@ class Database:
                 return Results(rows, False, cursor.description)
 
         with trace("sql", database=self.name, sql=sql.strip(), params=params):
-            results = await self.execute_against_connection_in_thread(
-                sql_operation_in_thread
-            )
+            results = await self.execute_fn(sql_operation_in_thread)
         return results
 
+    @property
+    def hash(self):
+        if self.cached_hash is not None:
+            return self.cached_hash
+        elif self.is_mutable or self.is_memory or self.is_temp_disk:
+            return None
+        elif self.ds.inspect_data and self.ds.inspect_data.get(self.name):
+            self.cached_hash = self.ds.inspect_data[self.name]["hash"]
+            return self.cached_hash
+        else:
+            p = Path(self.path)
+            self.cached_hash = inspect_hash(p)
+            return self.cached_hash
+
     @property
     def size(self):
-        if self.is_memory:
-            return 0
         if self.cached_size is not None:
             return self.cached_size
-        else:
+        elif self.is_memory:
+            return 0
+        elif self.is_mutable:
             return Path(self.path).stat().st_size
+        elif self.ds.inspect_data and self.ds.inspect_data.get(self.name):
+            self.cached_size = self.ds.inspect_data[self.name]["size"]
+            return self.cached_size
+        else:
+            self.cached_size = Path(self.path).stat().st_size
+            return self.cached_size
 
     async def table_counts(self, limit=10):
         if not self.is_mutable and self.cached_table_counts is not None:
@@ -186,7 +578,7 @@ class Database:
             try:
                 table_count = (
                     await self.execute(
-                        "select count(*) from [{}]".format(table),
+                        f"select count(*) from (select * from [{table}] limit {self.count_limit + 1})",
                         custom_time_limit=limit,
                     )
                 ).rows[0][0]
@@ -196,19 +588,27 @@ class Database:
             except (QueryInterrupted, sqlite3.OperationalError, sqlite3.DatabaseError):
                 counts[table] = None
         if not self.is_mutable:
-            self.cached_table_counts = counts
+            self._cached_table_counts = counts
         return counts
 
     @property
     def mtime_ns(self):
+        if self.is_memory:
+            return None
         return Path(self.path).stat().st_mtime_ns
 
-    @property
-    def name(self):
-        if self.is_memory:
-            return ":memory:"
-        else:
-            return Path(self.path).stem
+    async def attached_databases(self):
+        # This used to be:
+        #   select seq, name, file from pragma_database_list() where seq > 0
+        # But SQLite prior to 3.16.0 doesn't support pragma functions
+        results = await self.execute("PRAGMA database_list;")
+        # {'seq': 0, 'name': 'main', 'file': ''}
+        return [
+            AttachedDatabase(*row)
+            for row in results.rows
+            # Filter out the SQLite internal "temp" database, refs #2557
+            if row["seq"] > 0 and row["name"] != "temp"
+        ]
 
     async def table_exists(self, table):
         results = await self.execute(
@@ -216,72 +616,95 @@ class Database:
         )
         return bool(results.rows)
 
+    async def view_exists(self, table):
+        results = await self.execute(
+            "select 1 from sqlite_master where type='view' and name=?", params=(table,)
+        )
+        return bool(results.rows)
+
     async def table_names(self):
         results = await self.execute(
-            "select name from sqlite_master where type='table'"
+            "select name from sqlite_master where type='table' order by name"
         )
         return [r[0] for r in results.rows]
 
     async def table_columns(self, table):
-        return await self.execute_against_connection_in_thread(
-            lambda conn: table_columns(conn, table)
-        )
+        return await self.execute_fn(lambda conn: table_columns(conn, table))
+
+    async def table_column_details(self, table):
+        return await self.execute_fn(lambda conn: table_column_details(conn, table))
 
     async def primary_keys(self, table):
-        return await self.execute_against_connection_in_thread(
-            lambda conn: detect_primary_keys(conn, table)
-        )
+        return await self.execute_fn(lambda conn: detect_primary_keys(conn, table))
 
     async def fts_table(self, table):
-        return await self.execute_against_connection_in_thread(
-            lambda conn: detect_fts(conn, table)
-        )
+        return await self.execute_fn(lambda conn: detect_fts(conn, table))
 
     async def label_column_for_table(self, table):
-        explicit_label_column = self.ds.table_metadata(self.name, table).get(
+        explicit_label_column = (await self.ds.table_config(self.name, table)).get(
             "label_column"
         )
         if explicit_label_column:
             return explicit_label_column
-        # If a table has two columns, one of which is ID, then label_column is the other one
-        column_names = await self.execute_against_connection_in_thread(
-            lambda conn: table_columns(conn, table)
-        )
+
+        def column_details(conn):
+            # Returns {column_name: (type, is_unique)}
+            db = sqlite_utils.Database(conn)
+            columns = db[table].columns_dict
+            indexes = db[table].indexes
+            details = {}
+            for name in columns:
+                is_unique = any(
+                    index
+                    for index in indexes
+                    if index.columns == [name] and index.unique
+                )
+                details[name] = (columns[name], is_unique)
+            return details
+
+        column_details = await self.execute_fn(column_details)
+        # Is there just one unique column that's text?
+        unique_text_columns = [
+            name
+            for name, (type_, is_unique) in column_details.items()
+            if is_unique and type_ is str
+        ]
+        if len(unique_text_columns) == 1:
+            return unique_text_columns[0]
+
+        column_names = list(column_details.keys())
         # Is there a name or title column?
-        name_or_title = [c for c in column_names if c in ("name", "title")]
+        name_or_title = [c for c in column_names if c.lower() in ("name", "title")]
         if name_or_title:
             return name_or_title[0]
+        # If a table has two columns, one of which is ID, then label_column is the other one
         if (
             column_names
             and len(column_names) == 2
             and ("id" in column_names or "pk" in column_names)
+            and not set(column_names) == {"id", "pk"}
         ):
             return [c for c in column_names if c not in ("id", "pk")][0]
         # Couldn't find a label:
         return None
 
     async def foreign_keys_for_table(self, table):
-        return await self.execute_against_connection_in_thread(
+        return await self.execute_fn(
             lambda conn: get_outbound_foreign_keys(conn, table)
         )
 
     async def hidden_table_names(self):
-        # Mark tables 'hidden' if they relate to FTS virtual tables
-        hidden_tables = [
-            r[0]
-            for r in (
-                await self.execute(
-                    """
-                select name from sqlite_master
-                where rootpage = 0
-                and sql like '%VIRTUAL TABLE%USING FTS%'
-            """
-                )
-            ).rows
-        ]
-        has_spatialite = await self.execute_against_connection_in_thread(
-            detect_spatialite
-        )
+        hidden_tables = []
+        # Add any tables marked as hidden in config
+        db_config = self.ds.config.get("databases", {}).get(self.name, {})
+        if "tables" in db_config:
+            hidden_tables += [
+                t for t in db_config["tables"] if db_config["tables"][t].get("hidden")
+            ]
+
+        hidden_tables += await self.execute_fn(sqlite_hidden_table_names)
+
+        has_spatialite = await self.execute_fn(detect_spatialite)
         if has_spatialite:
             # Also hide Spatialite internal tables
             hidden_tables += [
@@ -294,33 +717,16 @@ class Database:
                 "sqlite_sequence",
                 "views_geometry_columns",
                 "virts_geometry_columns",
+                "data_licenses",
+                "KNN",
+                "KNN2",
             ] + [
-                r[0]
-                for r in (
-                    await self.execute(
-                        """
+                r[0] for r in (await self.execute("""
                         select name from sqlite_master
                         where name like "idx_%"
                         and type = "table"
-                    """
-                    )
-                ).rows
+                    """)).rows
             ]
-        # Add any from metadata.json
-        db_metadata = self.ds.metadata(database=self.name)
-        if "tables" in db_metadata:
-            hidden_tables += [
-                t
-                for t in db_metadata["tables"]
-                if db_metadata["tables"][t].get("hidden")
-            ]
-        # Also mark as hidden any tables which start with the name of a hidden table
-        # e.g. "searchable_fts" implies "searchable_fts_content" should be hidden
-        for table_name in await self.table_names():
-            for hidden_table in hidden_tables[:]:
-                if table_name.startswith(hidden_table):
-                    hidden_tables.append(table_name)
-                    continue
 
         return hidden_tables
 
@@ -329,12 +735,7 @@ class Database:
         return [r[0] for r in results.rows]
 
     async def get_all_foreign_keys(self):
-        return await self.execute_against_connection_in_thread(get_all_foreign_keys)
-
-    async def get_outbound_foreign_keys(self, table):
-        return await self.execute_against_connection_in_thread(
-            lambda conn: get_outbound_foreign_keys(conn, table)
-        )
+        return await self.execute_fn(get_all_foreign_keys)
 
     async def get_table_definition(self, table, type_="table"):
         table_definition_rows = list(
@@ -366,20 +767,143 @@ class Database:
             tags.append("mutable")
         if self.is_memory:
             tags.append("memory")
+        if self.is_temp_disk:
+            tags.append("temp_disk")
         if self.hash:
-            tags.append("hash={}".format(self.hash))
+            tags.append(f"hash={self.hash}")
         if self.size is not None:
-            tags.append("size={}".format(self.size))
+            tags.append(f"size={self.size}")
         tags_str = ""
         if tags:
-            tags_str = " ({})".format(", ".join(tags))
-        return "<Database: {}{}>".format(self.name, tags_str)
+            tags_str = f" ({', '.join(tags)})"
+        return f"<Database: {self.name}{tags_str}>"
+
+
+def _apply_write_wrapper(fn, wrapper_factory, track_event):
+    """Apply a single write_wrapper context manager around fn.
+
+    ``wrapper_factory`` is a callable that takes ``(conn)`` and optionally
+    ``track_event``, and returns a generator that yields exactly once.
+    Code before the yield runs before ``fn(conn)``, code after the yield
+    runs after.  The result of ``fn(conn)`` is sent into the generator
+    via ``.send()``, and any exception raised by ``fn(conn)`` is thrown
+    via ``.throw()``.
+    """
+
+    def wrapped(conn):
+        gen = call_with_supported_arguments(
+            wrapper_factory, conn=conn, track_event=track_event
+        )
+        # Advance to the yield point (run "before" code)
+        try:
+            next(gen)
+        except StopIteration:
+            # Generator didn't yield — just run fn unchanged
+            return fn(conn)
+
+        # Execute the actual write
+        try:
+            result = fn(conn)
+        except Exception:
+            # Throw exception into generator so it can handle it
+            try:
+                gen.throw(*sys.exc_info())
+            except StopIteration:
+                pass
+            # Re-raise the original exception
+            raise
+        else:
+            # Send the result back through the yield
+            try:
+                gen.send(result)
+            except StopIteration:
+                pass
+            return result
+
+    return wrapped
 
 
 class WriteTask:
-    __slots__ = ("fn", "task_id", "reply_queue")
+    __slots__ = (
+        "fn",
+        "task_id",
+        "loop",
+        "reply_future",
+        "isolated_connection",
+        "transaction",
+    )
 
-    def __init__(self, fn, task_id, reply_queue):
+    def __init__(
+        self, fn, task_id, loop, reply_future, isolated_connection, transaction
+    ):
         self.fn = fn
         self.task_id = task_id
-        self.reply_queue = reply_queue
+        self.loop = loop
+        self.reply_future = reply_future
+        self.isolated_connection = isolated_connection
+        self.transaction = transaction
+
+
+def _deliver_write_result(task, result, exception):
+    # Called from the write thread. Delivers the result back to the
+    # awaiting coroutine on its event loop via call_soon_threadsafe.
+    def _set():
+        if task.reply_future.done():
+            # Awaiter was cancelled; nothing to do.
+            return
+        if exception is not None:
+            task.reply_future.set_exception(exception)
+        else:
+            task.reply_future.set_result(result)
+
+    try:
+        task.loop.call_soon_threadsafe(_set)
+    except RuntimeError:
+        # Event loop has been closed; the awaiter is gone.
+        pass
+
+
+class QueryInterrupted(Exception):
+    def __init__(self, e, sql, params):
+        self.e = e
+        self.sql = sql
+        self.params = params
+
+    def __str__(self):
+        return "QueryInterrupted: {}".format(self.e)
+
+
+class MultipleValues(Exception):
+    pass
+
+
+class Results:
+    def __init__(self, rows, truncated, description):
+        self.rows = rows
+        self.truncated = truncated
+        self.description = description
+
+    @property
+    def columns(self):
+        return [d[0] for d in self.description]
+
+    def first(self):
+        if self.rows:
+            return self.rows[0]
+        else:
+            return None
+
+    def single_value(self):
+        if self.rows and 1 == len(self.rows) and 1 == len(self.rows[0]):
+            return self.rows[0][0]
+        else:
+            raise MultipleValues
+
+    def dicts(self):
+        return [dict(row) for row in self.rows]
+
+    def __iter__(self):
+        return iter(self.rows)
+
+    def __len__(self):
+        return len(self.rows)

datasette/default_actions.py

@@ -0,0 +1,133 @@
+from datasette import hookimpl
+from datasette.permissions import Action
+from datasette.resources import (
+    DatabaseResource,
+    TableResource,
+    QueryResource,
+)
+
+
+@hookimpl
+def register_actions():
+    """Register the core Datasette actions."""
+    return (
+        # Global actions (no resource_class)
+        Action(
+            name="view-instance",
+            abbr="vi",
+            description="View Datasette instance",
+        ),
+        Action(
+            name="permissions-debug",
+            abbr="pd",
+            description="Access permission debug tool",
+        ),
+        Action(
+            name="debug-menu",
+            abbr="dm",
+            description="View debug menu items",
+        ),
+        # Database-level actions (parent-level)
+        Action(
+            name="view-database",
+            abbr="vd",
+            description="View database",
+            resource_class=DatabaseResource,
+        ),
+        Action(
+            name="view-database-download",
+            abbr="vdd",
+            description="Download database file",
+            resource_class=DatabaseResource,
+            also_requires="view-database",
+        ),
+        Action(
+            name="execute-sql",
+            abbr="es",
+            description="Execute read-only SQL queries",
+            resource_class=DatabaseResource,
+            also_requires="view-database",
+        ),
+        Action(
+            name="execute-write-sql",
+            abbr="ews",
+            description="Execute writable SQL queries",
+            resource_class=DatabaseResource,
+            also_requires="view-database",
+        ),
+        Action(
+            name="create-table",
+            abbr="ct",
+            description="Create tables",
+            resource_class=DatabaseResource,
+        ),
+        Action(
+            name="store-query",
+            abbr="sq",
+            description="Create stored queries",
+            resource_class=DatabaseResource,
+            also_requires="execute-sql",
+        ),
+        # Table-level actions (child-level)
+        Action(
+            name="view-table",
+            abbr="vt",
+            description="View table",
+            resource_class=TableResource,
+        ),
+        Action(
+            name="insert-row",
+            abbr="ir",
+            description="Insert rows",
+            resource_class=TableResource,
+        ),
+        Action(
+            name="delete-row",
+            abbr="dr",
+            description="Delete rows",
+            resource_class=TableResource,
+        ),
+        Action(
+            name="update-row",
+            abbr="ur",
+            description="Update rows",
+            resource_class=TableResource,
+        ),
+        Action(
+            name="alter-table",
+            abbr="at",
+            description="Alter tables",
+            resource_class=TableResource,
+        ),
+        Action(
+            name="set-column-type",
+            abbr="sct",
+            description="Set column type",
+            resource_class=TableResource,
+        ),
+        Action(
+            name="drop-table",
+            abbr="dt",
+            description="Drop tables",
+            resource_class=TableResource,
+        ),
+        # Query-level actions (child-level)
+        Action(
+            name="view-query",
+            abbr="vq",
+            description="View named query results",
+            resource_class=QueryResource,
+        ),
+        Action(
+            name="update-query",
+            abbr="uq",
+            description="Update stored queries",
+            resource_class=QueryResource,
+        ),
+        Action(
+            name="delete-query",
+            abbr="dq",
+            description="Delete stored queries",
+            resource_class=QueryResource,
+        ),
+    )

datasette/default_column_types.py

@@ -0,0 +1,81 @@
+import json
+import re
+
+import markupsafe
+
+from datasette import hookimpl
+from datasette.column_types import ColumnType, SQLiteType
+
+
+class UrlColumnType(ColumnType):
+    name = "url"
+    description = "URL"
+    sqlite_types = (SQLiteType.TEXT,)
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        if not value or not isinstance(value, str):
+            return None
+        escaped = markupsafe.escape(value.strip())
+        return markupsafe.Markup(f'<a href="{escaped}">{escaped}</a>')
+
+    async def validate(self, value, datasette):
+        if value is None or value == "":
+            return None
+        if not isinstance(value, str):
+            return "URL must be a string"
+        if not re.match(r"^https?://\S+$", value.strip()):
+            return "Invalid URL"
+        return None
+
+
+class EmailColumnType(ColumnType):
+    name = "email"
+    description = "Email address"
+    sqlite_types = (SQLiteType.TEXT,)
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        if not value or not isinstance(value, str):
+            return None
+        escaped = markupsafe.escape(value.strip())
+        return markupsafe.Markup(f'<a href="mailto:{escaped}">{escaped}</a>')
+
+    async def validate(self, value, datasette):
+        if value is None or value == "":
+            return None
+        if not isinstance(value, str):
+            return "Email must be a string"
+        if not re.match(r"^[^@\s]+@[^@\s]+\.[^@\s]+$", value.strip()):
+            return "Invalid email address"
+        return None
+
+
+class JsonColumnType(ColumnType):
+    name = "json"
+    description = "JSON data"
+    sqlite_types = (SQLiteType.TEXT,)
+
+    async def render_cell(self, value, column, table, database, datasette, request):
+        if value is None:
+            return None
+        try:
+            parsed = json.loads(value) if isinstance(value, str) else value
+            formatted = json.dumps(parsed, indent=2)
+            escaped = markupsafe.escape(formatted)
+            return markupsafe.Markup(f"<pre>{escaped}</pre>")
+        except (json.JSONDecodeError, TypeError):
+            return None
+
+    async def validate(self, value, datasette):
+        if value is None or value == "":
+            return None
+        if isinstance(value, str):
+            try:
+                json.loads(value)
+            except json.JSONDecodeError:
+                return "Invalid JSON"
+        return None
+
+
+@hookimpl
+def register_column_types(datasette):
+    return [UrlColumnType, EmailColumnType, JsonColumnType]

datasette/default_database_actions.py

@@ -0,0 +1,24 @@
+from datasette import hookimpl
+from datasette.resources import DatabaseResource
+
+
+@hookimpl
+def database_actions(datasette, actor, database, request):
+    async def inner():
+        if not datasette.get_database(database).is_mutable:
+            return []
+        if not await datasette.allowed(
+            action="execute-write-sql",
+            resource=DatabaseResource(database),
+            actor=actor,
+        ):
+            return []
+        return [
+            {
+                "href": datasette.urls.database(database) + "/-/execute-write",
+                "label": "Execute write SQL",
+                "description": "Run writable SQL with table permission checks.",
+            }
+        ]
+
+    return inner

datasette/default_debug_menu.py

@@ -0,0 +1,75 @@
+from datasette import hookimpl
+from datasette.jump import JumpSQL
+
+DEBUG_MENU_ITEMS = (
+    (
+        "/-/databases",
+        "Databases",
+        "List of databases known to this Datasette instance.",
+    ),
+    (
+        "/-/plugins",
+        "Installed plugins",
+        "Review loaded plugins, their versions and their registered hooks.",
+    ),
+    (
+        "/-/versions",
+        "Version info",
+        "Check the Python, SQLite and dependency versions used by this server.",
+    ),
+    (
+        "/-/settings",
+        "Settings",
+        "Inspect the active Datasette settings and configuration values.",
+    ),
+    (
+        "/-/permissions",
+        "Debug permissions",
+        "Test permission checks for actors, actions and resources.",
+    ),
+    (
+        "/-/messages",
+        "Debug messages",
+        "Try out temporary flash messages shown to users.",
+    ),
+    (
+        "/-/allow-debug",
+        "Debug allow rules",
+        "Explore how allow blocks match actors against permission rules.",
+    ),
+    (
+        "/-/threads",
+        "Debug threads",
+        "Inspect worker threads and database tasks.",
+    ),
+    (
+        "/-/actor",
+        "Debug actor",
+        "View the actor object for the current signed-in user.",
+    ),
+    (
+        "/-/patterns",
+        "Pattern portfolio",
+        "Browse Datasette UI patterns.",
+    ),
+)
+
+
+@hookimpl
+def jump_items_sql(datasette, actor, request):
+    async def inner():
+        if not await datasette.allowed(action="debug-menu", actor=actor):
+            return []
+
+        return [
+            JumpSQL.menu_item(
+                label=label,
+                url=datasette.urls.path(path),
+                description=description,
+                search_text=f"debug {label} {description}",
+                item_type="debug",
+            )
+            for path, label, description in DEBUG_MENU_ITEMS
+        ]
+
+    return inner

datasette/default_jump_items.py

@@ -0,0 +1,82 @@
+from datasette import hookimpl
+from datasette.jump import JumpSQL
+
+
+@hookimpl
+def jump_items_sql(datasette, actor, request):
+    async def inner():
+        database_sql, database_params = await datasette.allowed_resources_sql(
+            action="view-database", actor=actor
+        )
+        table_sql, table_params = await datasette.allowed_resources_sql(
+            action="view-table", actor=actor
+        )
+        query_sql, query_params = await datasette.allowed_resources_sql(
+            action="view-query", actor=actor
+        )
+        return [
+            JumpSQL(
+                sql=f"""
+                WITH allowed_databases AS (
+                    {database_sql}
+                )
+                SELECT
+                    'database' AS type,
+                    parent AS label,
+                    NULL AS description,
+                    json_object(
+                        'method', 'database',
+                        'database', parent
+                    ) AS url,
+                    parent AS search_text,
+                    NULL AS display_name
+                FROM allowed_databases
+                """,
+                params=database_params,
+            ),
+            JumpSQL(
+                sql=f"""
+                WITH allowed_tables AS (
+                    {table_sql}
+                )
+                SELECT
+                    CASE WHEN catalog_views.view_name IS NULL THEN 'table' ELSE 'view' END AS type,
+                    allowed_tables.parent || ': ' || allowed_tables.child AS label,
+                    NULL AS description,
+                    json_object(
+                        'method', 'table',
+                        'database', allowed_tables.parent,
+                        'table', allowed_tables.child
+                    ) AS url,
+                    allowed_tables.parent || ' ' || allowed_tables.child AS search_text,
+                    NULL AS display_name
+                FROM allowed_tables
+                LEFT JOIN catalog_views
+                    ON catalog_views.database_name = allowed_tables.parent
+                   AND catalog_views.view_name = allowed_tables.child
+                """,
+                params=table_params,
+            ),
+            JumpSQL(
+                sql=f"""
+                WITH allowed_queries AS (
+                    {query_sql}
+                )
+                SELECT
+                    'query' AS type,
+                    allowed_queries.parent || ': ' || allowed_queries.child AS label,
+                    NULL AS description,
+                    json_object(
+                        'method', 'query',
+                        'database', allowed_queries.parent,
+                        'query', allowed_queries.child
+                    ) AS url,
+                    allowed_queries.parent || ' ' || allowed_queries.child AS search_text,
+                    NULL AS display_name
+                FROM allowed_queries
+                """,
+                params=query_params,
+            ),
+        ]
+
+    return inner

datasette/default_magic_parameters.py

@@ -0,0 +1,57 @@
+from datasette import hookimpl
+import datetime
+import os
+import time
+
+
+def header(key, request):
+    key = key.replace("_", "-").encode("utf-8")
+    headers_dict = dict(request.scope["headers"])
+    return headers_dict.get(key, b"").decode("utf-8")
+
+
+def actor(key, request):
+    if request.actor is None:
+        raise KeyError
+    return request.actor[key]
+
+
+def cookie(key, request):
+    return request.cookies[key]
+
+
+def now(key, request):
+    if key == "epoch":
+        return int(time.time())
+    elif key == "date_utc":
+        return datetime.datetime.now(datetime.timezone.utc).date().isoformat()
+    elif key == "datetime_utc":
+        return (
+            datetime.datetime.now(datetime.timezone.utc).strftime(r"%Y-%m-%dT%H:%M:%S")
+            + "Z"
+        )
+    else:
+        raise KeyError
+
+
+def random(key, request):
+    if key.startswith("chars_") and key.split("chars_")[-1].isdigit():
+        num_chars = int(key.split("chars_")[-1])
+        if num_chars % 2 == 1:
+            urandom_len = (num_chars + 1) / 2
+        else:
+            urandom_len = num_chars / 2
+        return os.urandom(int(urandom_len)).hex()[:num_chars]
+    else:
+        raise KeyError
+
+
+@hookimpl
+def register_magic_parameters():
+    return [
+        ("header", header),
+        ("actor", actor),
+        ("cookie", cookie),
+        ("now", now),
+        ("random", random),
+    ]

datasette/default_permissions/__init__.py

@@ -0,0 +1,34 @@
+"""
+Default permission implementations for Datasette.
+
+This module provides the built-in permission checking logic through implementations
+of the permission_resources_sql hook. The hooks are organized by their purpose:
+
+1. Actor Restrictions - Enforces _r allowlists embedded in actor tokens
+2. Root User - Grants full access when --root flag is used
+3. Config Rules - Applies permissions from datasette.yaml
+4. Default Settings - Enforces default_allow_sql and default view permissions
+
+IMPORTANT: These hooks return PermissionSQL objects that are combined using SQL
+UNION/INTERSECT operations. The order of evaluation is:
+  - restriction_sql fields are INTERSECTed (all must match)
+  - Regular sql fields are UNIONed and evaluated with cascading priority
+"""
+
+from __future__ import annotations
+
+# Re-export all hooks and public utilities
+from .restrictions import (
+    actor_restrictions_sql as actor_restrictions_sql,
+    restrictions_allow_action as restrictions_allow_action,
+    ActorRestrictions as ActorRestrictions,
+)
+from .root import root_user_permissions_sql as root_user_permissions_sql
+from .config import config_permissions_sql as config_permissions_sql
+from .defaults import (
+    # Avoid "datasette.default_permissions" does not explicitly export attribute
+    default_allow_sql_check as default_allow_sql_check,
+    default_action_permissions_sql as default_action_permissions_sql,
+    default_query_permissions_sql as default_query_permissions_sql,
+    DEFAULT_ALLOW_ACTIONS as DEFAULT_ALLOW_ACTIONS,
+)

datasette/default_permissions/config.py

@@ -0,0 +1,442 @@
+"""
+Config-based permission handling for Datasette.
+
+Applies permission rules from datasette.yaml configuration.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, List, Optional, Set, Tuple
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+from datasette.utils import actor_matches_allow
+
+from .helpers import PermissionRowCollector, get_action_name_variants
+
+
+class ConfigPermissionProcessor:
+    """
+    Processes permission rules from datasette.yaml configuration.
+
+    Configuration structure:
+
+    permissions:                    # Root-level permissions block
+      view-instance:
+        id: admin
+
+    databases:
+      mydb:
+        permissions:                # Database-level permissions
+          view-database:
+            id: admin
+        allow:                      # Database-level allow block (for view-*)
+          id: viewer
+        allow_sql:                  # execute-sql allow block
+          id: analyst
+        tables:
+          users:
+            permissions:            # Table-level permissions
+              view-table:
+                id: admin
+            allow:                  # Table-level allow block
+              id: viewer
+        queries:
+          my_query:
+            permissions:            # Query-level permissions
+              view-query:
+                id: admin
+            allow:                  # Query-level allow block
+              id: viewer
+    """
+
+    def __init__(
+        self,
+        datasette: "Datasette",
+        actor: Optional[dict],
+        action: str,
+    ):
+        self.datasette = datasette
+        self.actor = actor
+        self.action = action
+        self.config = datasette.config or {}
+        self.collector = PermissionRowCollector(prefix="cfg")
+
+        # Pre-compute action variants
+        self.action_checks = get_action_name_variants(datasette, action)
+        self.action_obj = datasette.actions.get(action)
+
+        # Parse restrictions if present
+        self.has_restrictions = actor and "_r" in actor if actor else False
+        self.restrictions = actor.get("_r", {}) if actor else {}
+
+        # Pre-compute restriction info for efficiency
+        self.restricted_databases: Set[str] = set()
+        self.restricted_tables: Set[Tuple[str, str]] = set()
+
+        if self.has_restrictions:
+            self.restricted_databases = {
+                db_name
+                for db_name, db_actions in (self.restrictions.get("d") or {}).items()
+                if self.action_checks.intersection(db_actions)
+            }
+            self.restricted_tables = {
+                (db_name, table_name)
+                for db_name, tables in (self.restrictions.get("r") or {}).items()
+                for table_name, table_actions in tables.items()
+                if self.action_checks.intersection(table_actions)
+            }
+            # Tables implicitly reference their parent databases
+            self.restricted_databases.update(db for db, _ in self.restricted_tables)
+
+    def evaluate_allow_block(self, allow_block: Any) -> Optional[bool]:
+        """Evaluate an allow block against the current actor."""
+        if allow_block is None:
+            return None
+        return actor_matches_allow(self.actor, allow_block)
+
+    def is_in_restriction_allowlist(
+        self,
+        parent: Optional[str],
+        child: Optional[str],
+    ) -> bool:
+        """Check if resource is allowed by actor restrictions."""
+        if not self.has_restrictions:
+            return True  # No restrictions, all resources allowed
+
+        # Check global allowlist
+        if self.action_checks.intersection(self.restrictions.get("a", [])):
+            return True
+
+        # Check database-level allowlist
+        if parent and self.action_checks.intersection(
+            self.restrictions.get("d", {}).get(parent, [])
+        ):
+            return True
+
+        # Check table-level allowlist
+        if parent:
+            table_restrictions = (self.restrictions.get("r", {}) or {}).get(parent, {})
+            if child:
+                table_actions = table_restrictions.get(child, [])
+                if self.action_checks.intersection(table_actions):
+                    return True
+            else:
+                # Parent query should proceed if any child in this database is allowlisted
+                for table_actions in table_restrictions.values():
+                    if self.action_checks.intersection(table_actions):
+                        return True
+
+        # Parent/child both None: include if any restrictions exist for this action
+        if parent is None and child is None:
+            if self.action_checks.intersection(self.restrictions.get("a", [])):
+                return True
+            if self.restricted_databases:
+                return True
+            if self.restricted_tables:
+                return True
+
+        return False
+
+    def add_permissions_rule(
+        self,
+        parent: Optional[str],
+        child: Optional[str],
+        permissions_block: Optional[dict],
+        scope_desc: str,
+    ) -> None:
+        """Add a rule from a permissions:{action} block."""
+        if permissions_block is None:
+            return
+
+        action_allow_block = permissions_block.get(self.action)
+        result = self.evaluate_allow_block(action_allow_block)
+
+        self.collector.add(
+            parent=parent,
+            child=child,
+            allow=result,
+            reason=f"config {'allow' if result else 'deny'} {scope_desc}",
+            if_not_none=True,
+        )
+
+    def add_allow_block_rule(
+        self,
+        parent: Optional[str],
+        child: Optional[str],
+        allow_block: Any,
+        scope_desc: str,
+    ) -> None:
+        """
+        Add rules from an allow:{} block.
+
+        For allow blocks, if the block exists but doesn't match the actor,
+        this is treated as a deny. We also handle the restriction-gate logic.
+        """
+        if allow_block is None:
+            return
+
+        # Skip if resource is not in restriction allowlist
+        if not self.is_in_restriction_allowlist(parent, child):
+            return
+
+        result = self.evaluate_allow_block(allow_block)
+        bool_result = bool(result)
+
+        self.collector.add(
+            parent,
+            child,
+            bool_result,
+            f"config {'allow' if result else 'deny'} {scope_desc}",
+        )
+
+        # Handle restriction-gate: add explicit denies for restricted resources
+        self._add_restriction_gate_denies(parent, child, bool_result, scope_desc)
+
+    def _add_restriction_gate_denies(
+        self,
+        parent: Optional[str],
+        child: Optional[str],
+        is_allowed: bool,
+        scope_desc: str,
+    ) -> None:
+        """
+        When a config rule denies at a higher level, add explicit denies
+        for restricted resources to prevent child-level allows from
+        incorrectly granting access.
+        """
+        if is_allowed or child is not None or not self.has_restrictions:
+            return
+
+        if not self.action_obj:
+            return
+
+        reason = f"config deny {scope_desc} (restriction gate)"
+
+        if parent is None:
+            # Root-level deny: add denies for all restricted resources
+            if self.action_obj.takes_parent:
+                for db_name in self.restricted_databases:
+                    self.collector.add(db_name, None, False, reason)
+            if self.action_obj.takes_child:
+                for db_name, table_name in self.restricted_tables:
+                    self.collector.add(db_name, table_name, False, reason)
+        else:
+            # Database-level deny: add denies for tables in that database
+            if self.action_obj.takes_child:
+                for db_name, table_name in self.restricted_tables:
+                    if db_name == parent:
+                        self.collector.add(db_name, table_name, False, reason)
+
+    def process(self) -> Optional[PermissionSQL]:
+        """Process all config rules and return combined PermissionSQL."""
+        self._process_root_permissions()
+        self._process_databases()
+        self._process_root_allow_blocks()
+
+        return self.collector.to_permission_sql()
+
+    def _process_root_permissions(self) -> None:
+        """Process root-level permissions block."""
+        root_perms = self.config.get("permissions") or {}
+        self.add_permissions_rule(
+            None,
+            None,
+            root_perms,
+            f"permissions for {self.action}",
+        )
+
+    def _process_databases(self) -> None:
+        """Process database-level and nested configurations."""
+        databases = self.config.get("databases") or {}
+
+        for db_name, db_config in databases.items():
+            self._process_database(db_name, db_config or {})
+
+    def _process_database(self, db_name: str, db_config: dict) -> None:
+        """Process a single database's configuration."""
+        # Database-level permissions block
+        db_perms = db_config.get("permissions") or {}
+        self.add_permissions_rule(
+            db_name,
+            None,
+            db_perms,
+            f"permissions for {self.action} on {db_name}",
+        )
+
+        # Process tables
+        for table_name, table_config in (db_config.get("tables") or {}).items():
+            self._process_table(db_name, table_name, table_config or {})
+
+        # Process queries
+        for query_name, query_config in (db_config.get("queries") or {}).items():
+            self._process_query(db_name, query_name, query_config)
+
+        # Database-level allow blocks
+        self._process_database_allow_blocks(db_name, db_config)
+
+    def _process_table(
+        self,
+        db_name: str,
+        table_name: str,
+        table_config: dict,
+    ) -> None:
+        """Process a single table's configuration."""
+        # Table-level permissions block
+        table_perms = table_config.get("permissions") or {}
+        self.add_permissions_rule(
+            db_name,
+            table_name,
+            table_perms,
+            f"permissions for {self.action} on {db_name}/{table_name}",
+        )
+
+        # Table-level allow block (for view-table)
+        if self.action == "view-table":
+            self.add_allow_block_rule(
+                db_name,
+                table_name,
+                table_config.get("allow"),
+                f"allow for {self.action} on {db_name}/{table_name}",
+            )
+
+    def _process_query(
+        self,
+        db_name: str,
+        query_name: str,
+        query_config: Any,
+    ) -> None:
+        """Process a single query's configuration."""
+        # Query config can be a string (just SQL) or dict
+        if not isinstance(query_config, dict):
+            return
+
+        # Query-level permissions block
+        query_perms = query_config.get("permissions") or {}
+        self.add_permissions_rule(
+            db_name,
+            query_name,
+            query_perms,
+            f"permissions for {self.action} on {db_name}/{query_name}",
+        )
+
+        # Query-level allow block (for view-query)
+        if self.action == "view-query":
+            self.add_allow_block_rule(
+                db_name,
+                query_name,
+                query_config.get("allow"),
+                f"allow for {self.action} on {db_name}/{query_name}",
+            )
+
+    def _process_database_allow_blocks(
+        self,
+        db_name: str,
+        db_config: dict,
+    ) -> None:
+        """Process database-level allow/allow_sql blocks."""
+        # view-database allow block
+        if self.action == "view-database":
+            self.add_allow_block_rule(
+                db_name,
+                None,
+                db_config.get("allow"),
+                f"allow for {self.action} on {db_name}",
+            )
+
+        # execute-sql allow_sql block
+        if self.action == "execute-sql":
+            self.add_allow_block_rule(
+                db_name,
+                None,
+                db_config.get("allow_sql"),
+                f"allow_sql for {db_name}",
+            )
+
+        # view-table uses database-level allow for inheritance
+        if self.action == "view-table":
+            self.add_allow_block_rule(
+                db_name,
+                None,
+                db_config.get("allow"),
+                f"allow for {self.action} on {db_name}",
+            )
+
+        # view-query uses database-level allow for inheritance
+        if self.action == "view-query":
+            self.add_allow_block_rule(
+                db_name,
+                None,
+                db_config.get("allow"),
+                f"allow for {self.action} on {db_name}",
+            )
+
+    def _process_root_allow_blocks(self) -> None:
+        """Process root-level allow/allow_sql blocks."""
+        root_allow = self.config.get("allow")
+
+        if self.action == "view-instance":
+            self.add_allow_block_rule(
+                None,
+                None,
+                root_allow,
+                "allow for view-instance",
+            )
+
+        if self.action == "view-database":
+            self.add_allow_block_rule(
+                None,
+                None,
+                root_allow,
+                "allow for view-database",
+            )
+
+        if self.action == "view-table":
+            self.add_allow_block_rule(
+                None,
+                None,
+                root_allow,
+                "allow for view-table",
+            )
+
+        if self.action == "view-query":
+            self.add_allow_block_rule(
+                None,
+                None,
+                root_allow,
+                "allow for view-query",
+            )
+
+        if self.action == "execute-sql":
+            self.add_allow_block_rule(
+                None,
+                None,
+                self.config.get("allow_sql"),
+                "allow_sql",
+            )
+
+
+@hookimpl(specname="permission_resources_sql")
+async def config_permissions_sql(
+    datasette: "Datasette",
+    actor: Optional[dict],
+    action: str,
+) -> Optional[List[PermissionSQL]]:
+    """
+    Apply permission rules from datasette.yaml configuration.
+
+    This processes:
+    - permissions: blocks at root, database, table, and query levels
+    - allow: blocks for view-* actions
+    - allow_sql: blocks for execute-sql action
+    """
+    processor = ConfigPermissionProcessor(datasette, actor, action)
+    result = processor.process()
+
+    if result is None:
+        return []
+
+    return [result]

datasette/default_permissions/defaults.py

@@ -0,0 +1,114 @@
+"""
+Default permission settings for Datasette.
+
+Provides default allow rules for standard view/execute actions.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+# Actions that are allowed by default (unless --default-deny is used)
+DEFAULT_ALLOW_ACTIONS = frozenset(
+    {
+        "view-instance",
+        "view-database",
+        "view-database-download",
+        "view-table",
+        "view-query",
+        "execute-sql",
+    }
+)
+
+
+@hookimpl(specname="permission_resources_sql")
+async def default_allow_sql_check(
+    datasette: "Datasette",
+    actor: Optional[dict],
+    action: str,
+) -> Optional[PermissionSQL]:
+    """
+    Enforce the default_allow_sql setting.
+
+    When default_allow_sql is false (the default), execute-sql is denied
+    unless explicitly allowed by config or other rules.
+    """
+    if action == "execute-sql":
+        if not datasette.setting("default_allow_sql"):
+            return PermissionSQL.deny(reason="default_allow_sql is false")
+
+    return None
+
+
+@hookimpl(specname="permission_resources_sql")
+async def default_action_permissions_sql(
+    datasette: "Datasette",
+    actor: Optional[dict],
+    action: str,
+) -> Optional[PermissionSQL]:
+    """
+    Provide default allow rules for standard view/execute actions.
+
+    These defaults are skipped when datasette is started with --default-deny.
+    The restriction_sql mechanism (from actor_restrictions_sql) will still
+    filter these results if the actor has restrictions.
+    """
+    if datasette.default_deny:
+        return None
+
+    if action in DEFAULT_ALLOW_ACTIONS:
+        reason = f"default allow for {action}".replace("'", "''")
+        return PermissionSQL.allow(reason=reason)
+
+    return None
+
+
+@hookimpl(specname="permission_resources_sql")
+async def default_query_permissions_sql(
+    datasette: "Datasette",
+    actor: Optional[dict],
+    action: str,
+) -> Optional[PermissionSQL]:
+    actor_id = actor.get("id") if isinstance(actor, dict) else None
+
+    if action not in {"view-query", "update-query", "delete-query"}:
+        return None
+
+    params = {"query_owner_id": actor_id}
+    rule_sqls = []
+    if actor_id is not None:
+        if action in {"update-query", "delete-query"}:
+            # Query owner can update/delete query
+            rule_sqls.append("""
+            SELECT database_name AS parent, name AS child, 1 AS allow,
+              'query owner' AS reason
+            FROM queries
+            WHERE source = 'user'
+              AND owner_id = :query_owner_id
+            """)
+        else:
+            # Query owner can view-query
+            rule_sqls.append("""
+            SELECT database_name AS parent, name AS child, 1 AS allow,
+              'query owner' AS reason
+            FROM queries
+            WHERE owner_id = :query_owner_id
+            """)
+
+    # restriction_sql enforces private queries ONLY visible/mutable by owner
+    return PermissionSQL(
+        sql="\nUNION ALL\n".join(rule_sqls) if rule_sqls else None,
+        restriction_sql="""
+        SELECT database_name AS parent, name AS child
+        FROM queries
+        WHERE is_private = 0
+           OR owner_id = :query_owner_id
+        """,
+        params=params,
+    )

datasette/default_permissions/helpers.py

@@ -0,0 +1,85 @@
+"""
+Shared helper utilities for default permission implementations.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Optional, Set
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette.permissions import PermissionSQL
+
+
+def get_action_name_variants(datasette: "Datasette", action: str) -> Set[str]:
+    """
+    Get all name variants for an action (full name and abbreviation).
+
+    Example:
+        get_action_name_variants(ds, "view-table") -> {"view-table", "vt"}
+    """
+    variants = {action}
+    action_obj = datasette.actions.get(action)
+    if action_obj and action_obj.abbr:
+        variants.add(action_obj.abbr)
+    return variants
+
+
+def action_in_list(datasette: "Datasette", action: str, action_list: list) -> bool:
+    """Check if an action (or its abbreviation) is in a list."""
+    return bool(get_action_name_variants(datasette, action).intersection(action_list))
+
+
+@dataclass
+class PermissionRow:
+    """A single permission rule row."""
+
+    parent: Optional[str]
+    child: Optional[str]
+    allow: bool
+    reason: str
+
+
+class PermissionRowCollector:
+    """Collects permission rows and converts them to PermissionSQL."""
+
+    def __init__(self, prefix: str = "row"):
+        self.rows: List[PermissionRow] = []
+        self.prefix = prefix
+
+    def add(
+        self,
+        parent: Optional[str],
+        child: Optional[str],
+        allow: Optional[bool],
+        reason: str,
+        if_not_none: bool = False,
+    ) -> None:
+        """Add a permission row. If if_not_none=True, only add if allow is not None."""
+        if if_not_none and allow is None:
+            return
+        self.rows.append(PermissionRow(parent, child, allow, reason))
+
+    def to_permission_sql(self) -> Optional[PermissionSQL]:
+        """Convert collected rows to a PermissionSQL object."""
+        if not self.rows:
+            return None
+
+        parts = []
+        params = {}
+
+        for idx, row in enumerate(self.rows):
+            key = f"{self.prefix}_{idx}"
+            parts.append(
+                f"SELECT :{key}_parent AS parent, :{key}_child AS child, "
+                f":{key}_allow AS allow, :{key}_reason AS reason"
+            )
+            params[f"{key}_parent"] = row.parent
+            params[f"{key}_child"] = row.child
+            params[f"{key}_allow"] = 1 if row.allow else 0
+            params[f"{key}_reason"] = row.reason
+
+        sql = "\nUNION ALL\n".join(parts)
+        return PermissionSQL(sql=sql, params=params)

datasette/default_permissions/restrictions.py

@@ -0,0 +1,195 @@
+"""
+Actor restriction handling for Datasette permissions.
+
+This module handles the _r (restrictions) key in actor dictionaries, which
+contains allowlists of resources the actor can access.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Optional, Set, Tuple
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+from .helpers import action_in_list, get_action_name_variants
+
+
+@dataclass
+class ActorRestrictions:
+    """Parsed actor restrictions from the _r key."""
+
+    global_actions: List[str]  # _r.a - globally allowed actions
+    database_actions: dict  # _r.d - {db_name: [actions]}
+    table_actions: dict  # _r.r - {db_name: {table: [actions]}}
+
+    @classmethod
+    def from_actor(cls, actor: Optional[dict]) -> Optional["ActorRestrictions"]:
+        """Parse restrictions from actor dict. Returns None if no restrictions."""
+        if not actor:
+            return None
+        assert isinstance(actor, dict), "actor must be a dictionary"
+
+        restrictions = actor.get("_r")
+        if restrictions is None:
+            return None
+
+        return cls(
+            global_actions=restrictions.get("a", []),
+            database_actions=restrictions.get("d", {}),
+            table_actions=restrictions.get("r", {}),
+        )
+
+    def is_action_globally_allowed(self, datasette: "Datasette", action: str) -> bool:
+        """Check if action is in the global allowlist."""
+        return action_in_list(datasette, action, self.global_actions)
+
+    def get_allowed_databases(self, datasette: "Datasette", action: str) -> Set[str]:
+        """Get database names where this action is allowed."""
+        allowed = set()
+        for db_name, db_actions in self.database_actions.items():
+            if action_in_list(datasette, action, db_actions):
+                allowed.add(db_name)
+        return allowed
+
+    def get_allowed_tables(
+        self, datasette: "Datasette", action: str
+    ) -> Set[Tuple[str, str]]:
+        """Get (database, table) pairs where this action is allowed."""
+        allowed = set()
+        for db_name, tables in self.table_actions.items():
+            for table_name, table_actions in tables.items():
+                if action_in_list(datasette, action, table_actions):
+                    allowed.add((db_name, table_name))
+        return allowed
+
+
+@hookimpl(specname="permission_resources_sql")
+async def actor_restrictions_sql(
+    datasette: "Datasette",
+    actor: Optional[dict],
+    action: str,
+) -> Optional[List[PermissionSQL]]:
+    """
+    Handle actor restriction-based permission rules.
+
+    When an actor has an "_r" key, it contains an allowlist of resources they
+    can access. This function returns restriction_sql that filters the final
+    results to only include resources in that allowlist.
+
+    The _r structure:
+    {
+        "a": ["vi", "pd"],           # Global actions allowed
+        "d": {"mydb": ["vt", "es"]}, # Database-level actions
+        "r": {"mydb": {"users": ["vt"]}}  # Table-level actions
+    }
+    """
+    if not actor:
+        return None
+
+    restrictions = ActorRestrictions.from_actor(actor)
+
+    if restrictions is None:
+        # No restrictions - all resources allowed
+        return []
+
+    # If globally allowed, no filtering needed
+    if restrictions.is_action_globally_allowed(datasette, action):
+        return []
+
+    # Build restriction SQL
+    allowed_dbs = restrictions.get_allowed_databases(datasette, action)
+    allowed_tables = restrictions.get_allowed_tables(datasette, action)
+
+    # If nothing is allowed for this action, return empty-set restriction
+    if not allowed_dbs and not allowed_tables:
+        return [
+            PermissionSQL(
+                params={"deny": f"actor restrictions: {action} not in allowlist"},
+                restriction_sql="SELECT NULL AS parent, NULL AS child WHERE 0",
+            )
+        ]
+
+    # Build UNION of allowed resources
+    selects = []
+    params = {}
+    counter = 0
+
+    # Database-level entries (parent, NULL) - allows all children
+    for db_name in allowed_dbs:
+        key = f"restr_{counter}"
+        counter += 1
+        selects.append(f"SELECT :{key}_parent AS parent, NULL AS child")
+        params[f"{key}_parent"] = db_name
+
+    # Table-level entries (parent, child)
+    for db_name, table_name in allowed_tables:
+        key = f"restr_{counter}"
+        counter += 1
+        selects.append(f"SELECT :{key}_parent AS parent, :{key}_child AS child")
+        params[f"{key}_parent"] = db_name
+        params[f"{key}_child"] = table_name
+
+    restriction_sql = "\nUNION ALL\n".join(selects)
+
+    return [PermissionSQL(params=params, restriction_sql=restriction_sql)]
+
+
+def restrictions_allow_action(
+    datasette: "Datasette",
+    restrictions: dict,
+    action: str,
+    resource: Optional[str | Tuple[str, str]],
+) -> bool:
+    """
+    Check if restrictions allow the requested action on the requested resource.
+
+    This is a synchronous utility function for use by other code that needs
+    to quickly check restriction allowlists.
+
+    Args:
+        datasette: The Datasette instance
+        restrictions: The _r dict from an actor
+        action: The action name to check
+        resource: None for global, str for database, (db, table) tuple for table
+
+    Returns:
+        True if allowed, False if denied
+    """
+    # Does this action have an abbreviation?
+    to_check = get_action_name_variants(datasette, action)
+
+    # Check global level (any resource)
+    all_allowed = restrictions.get("a")
+    if all_allowed is not None:
+        assert isinstance(all_allowed, list)
+        if to_check.intersection(all_allowed):
+            return True
+
+    # Check database level
+    if resource:
+        if isinstance(resource, str):
+            database_name = resource
+        else:
+            database_name = resource[0]
+        database_allowed = restrictions.get("d", {}).get(database_name)
+        if database_allowed is not None:
+            assert isinstance(database_allowed, list)
+            if to_check.intersection(database_allowed):
+                return True
+
+    # Check table/resource level
+    if resource is not None and not isinstance(resource, str) and len(resource) == 2:
+        database, table = resource
+        table_allowed = restrictions.get("r", {}).get(database, {}).get(table)
+        if table_allowed is not None:
+            assert isinstance(table_allowed, list)
+            if to_check.intersection(table_allowed):
+                return True
+
+    # This action is not explicitly allowed, so reject it
+    return False

datasette/default_permissions/root.py

@@ -0,0 +1,29 @@
+"""
+Root user permission handling for Datasette.
+
+Grants full permissions to the root user when --root flag is used.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette import hookimpl
+from datasette.permissions import PermissionSQL
+
+
+@hookimpl(specname="permission_resources_sql")
+async def root_user_permissions_sql(
+    datasette: "Datasette",
+    actor: Optional[dict],
+) -> Optional[PermissionSQL]:
+    """
+    Grant root user full permissions when --root flag is used.
+    """
+    if not datasette.root_enabled:
+        return None
+    if actor is not None and actor.get("id") == "root":
+        return PermissionSQL.allow(reason="root user")

datasette/default_permissions/tokens.py

@@ -0,0 +1,40 @@
+"""
+Token authentication for Datasette.
+
+Registers the default SignedTokenHandler and delegates token verification
+to datasette.verify_token() so all registered handlers are tried.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from datasette.app import Datasette
+
+from datasette import hookimpl
+from datasette.tokens import SignedTokenHandler
+
+
+@hookimpl
+def register_token_handler(datasette: "Datasette"):
+    """Register the default signed token handler."""
+    return SignedTokenHandler()
+
+
+@hookimpl(specname="actor_from_request")
+async def actor_from_signed_api_token(
+    datasette: "Datasette", request
+) -> Optional[dict]:
+    """
+    Authenticate requests using API tokens by delegating to all registered
+    token handlers via datasette.verify_token().
+    """
+    authorization = request.headers.get("authorization")
+    if not authorization:
+        return None
+    if not authorization.startswith("Bearer "):
+        return None
+
+    token = authorization[len("Bearer ") :]
+    return await datasette.verify_token(token)

datasette/events.py

@@ -0,0 +1,293 @@
+from abc import ABC, abstractproperty
+from dataclasses import asdict, dataclass, field
+from datasette.hookspecs import hookimpl
+from datetime import datetime, timezone
+
+
+@dataclass
+class Event(ABC):
+    @abstractproperty
+    def name(self):
+        pass
+
+    created: datetime = field(
+        init=False, default_factory=lambda: datetime.now(timezone.utc)
+    )
+    actor: dict | None
+
+    def properties(self):
+        properties = asdict(self)
+        properties.pop("actor", None)
+        properties.pop("created", None)
+        return properties
+
+
+@dataclass
+class LoginEvent(Event):
+    """
+    Event name: ``login``
+
+    A user (represented by ``event.actor``) has logged in.
+    """
+
+    name = "login"
+
+
+@dataclass
+class LogoutEvent(Event):
+    """
+    Event name: ``logout``
+
+    A user (represented by ``event.actor``) has logged out.
+    """
+
+    name = "logout"
+
+
+@dataclass
+class CreateTokenEvent(Event):
+    """
+    Event name: ``create-token``
+
+    A user created an API token.
+
+    :ivar expires_after: Number of seconds after which this token will expire.
+    :type expires_after: int or None
+    :ivar restrict_all: Restricted permissions for this token.
+    :type restrict_all: list
+    :ivar restrict_database: Restricted database permissions for this token.
+    :type restrict_database: dict
+    :ivar restrict_resource: Restricted resource permissions for this token.
+    :type restrict_resource: dict
+    """
+
+    name = "create-token"
+    expires_after: int | None
+    restrict_all: list
+    restrict_database: dict
+    restrict_resource: dict
+
+
+@dataclass
+class CreateTableEvent(Event):
+    """
+    Event name: ``create-table``
+
+    A new table has been created in the database.
+
+    :ivar database: The name of the database where the table was created.
+    :type database: str
+    :ivar table: The name of the table that was created
+    :type table: str
+    :ivar schema: The SQL schema definition for the new table.
+    :type schema: str
+    """
+
+    name = "create-table"
+    database: str
+    table: str
+    schema: str
+
+
+@dataclass
+class DropTableEvent(Event):
+    """
+    Event name: ``drop-table``
+
+    A table has been dropped from the database.
+
+    :ivar database: The name of the database where the table was dropped.
+    :type database: str
+    :ivar table: The name of the table that was dropped
+    :type table: str
+    """
+
+    name = "drop-table"
+    database: str
+    table: str
+
+
+@dataclass
+class AlterTableEvent(Event):
+    """
+    Event name: ``alter-table``
+
+    A table has been altered.
+
+    :ivar database: The name of the database where the table was altered
+    :type database: str
+    :ivar table: The name of the table that was altered
+    :type table: str
+    :ivar before_schema: The table's SQL schema before the alteration
+    :type before_schema: str
+    :ivar after_schema: The table's SQL schema after the alteration
+    :type after_schema: str
+    """
+
+    name = "alter-table"
+    database: str
+    table: str
+    before_schema: str
+    after_schema: str
+
+
+@dataclass
+class InsertRowsEvent(Event):
+    """
+    Event name: ``insert-rows``
+
+    Rows were inserted into a table.
+
+    :ivar database: The name of the database where the rows were inserted.
+    :type database: str
+    :ivar table: The name of the table where the rows were inserted.
+    :type table: str
+    :ivar num_rows: The number of rows that were requested to be inserted.
+    :type num_rows: int
+    :ivar ignore: Was ignore set?
+    :type ignore: bool
+    :ivar replace: Was replace set?
+    :type replace: bool
+    """
+
+    name = "insert-rows"
+    database: str
+    table: str
+    num_rows: int
+    ignore: bool
+    replace: bool
+
+
+@dataclass
+class UpsertRowsEvent(Event):
+    """
+    Event name: ``upsert-rows``
+
+    Rows were upserted into a table.
+
+    :ivar database: The name of the database where the rows were inserted.
+    :type database: str
+    :ivar table: The name of the table where the rows were inserted.
+    :type table: str
+    :ivar num_rows: The number of rows that were requested to be inserted.
+    :type num_rows: int
+    """
+
+    name = "upsert-rows"
+    database: str
+    table: str
+    num_rows: int
+
+
+@dataclass
+class UpdateRowEvent(Event):
+    """
+    Event name: ``update-row``
+
+    A row was updated in a table.
+
+    :ivar database: The name of the database where the row was updated.
+    :type database: str
+    :ivar table: The name of the table where the row was updated.
+    :type table: str
+    :ivar pks: The primary key values of the updated row.
+    """
+
+    name = "update-row"
+    database: str
+    table: str
+    pks: list
+
+
+@dataclass
+class RenameTableEvent(Event):
+    """
+    Event name: ``rename-table``
+
+    A table has been renamed.
+
+    :ivar database: The name of the database containing the renamed table.
+    :type database: str
+    :ivar old_table: The previous name of the table.
+    :type old_table: str
+    :ivar new_table: The new name of the table.
+    :type new_table: str
+    """
+
+    name = "rename-table"
+    database: str
+    old_table: str
+    new_table: str
+
+
+@dataclass
+class DeleteRowEvent(Event):
+    """
+    Event name: ``delete-row``
+
+    A row was deleted from a table.
+
+    :ivar database: The name of the database where the row was deleted.
+    :type database: str
+    :ivar table: The name of the table where the row was deleted.
+    :type table: str
+    :ivar pks: The primary key values of the deleted row.
+    """
+
+    name = "delete-row"
+    database: str
+    table: str
+    pks: list
+
+
+@hookimpl
+def write_wrapper(datasette, database, request, transaction):
+    def wrapper(conn, track_event):
+        # Snapshot rootpage -> name before the write
+        before = {
+            row[1]: row[0]
+            for row in conn.execute(
+                "select name, rootpage from sqlite_master"
+                " where type='table' and rootpage != 0"
+            ).fetchall()
+        }
+        yield
+        # Snapshot rootpage -> name after the write
+        after = {
+            row[1]: row[0]
+            for row in conn.execute(
+                "select name, rootpage from sqlite_master"
+                " where type='table' and rootpage != 0"
+            ).fetchall()
+        }
+        # Detect renames: same rootpage, different name
+        for rootpage, old_name in before.items():
+            new_name = after.get(rootpage)
+            if new_name and new_name != old_name:
+                track_event(
+                    RenameTableEvent(
+                        actor=request.actor if request else None,
+                        database=database,
+                        old_table=old_name,
+                        new_table=new_name,
+                    )
+                )
+
+    return wrapper
+
+
+@hookimpl
+def register_events():
+    return [
+        LoginEvent,
+        LogoutEvent,
+        CreateTableEvent,
+        CreateTokenEvent,
+        AlterTableEvent,
+        RenameTableEvent,
+        DropTableEvent,
+        InsertRowsEvent,
+        UpsertRowsEvent,
+        UpdateRowEvent,
+        DeleteRowEvent,
+    ]

datasette/facets.py

@@ -1,20 +1,18 @@
 import json
 import urllib
-import re
 from datasette import hookimpl
+from datasette.database import QueryInterrupted
 from datasette.utils import (
     escape_sqlite,
     path_with_added_args,
     path_with_removed_args,
     detect_json1,
-    QueryInterrupted,
-    InvalidSql,
     sqlite3,
 )
 
 
-def load_facet_configs(request, table_metadata):
-    # Given a request and the metadata configuration for a table, return
+def load_facet_configs(request, table_config):
+    # Given a request and the configuration for a table, return
     # a dictionary of selected facets, their lists of configs and for each
     # config whether it came from the request or the metadata.
     #
@@ -22,21 +20,21 @@ def load_facet_configs(request, table_metadata):
     #       {"source": "metadata", "config": config1},
     #       {"source": "request", "config": config2}]}
     facet_configs = {}
-    table_metadata = table_metadata or {}
-    metadata_facets = table_metadata.get("facets", [])
-    for metadata_config in metadata_facets:
-        if isinstance(metadata_config, str):
+    table_config = table_config or {}
+    table_facet_configs = table_config.get("facets", [])
+    for facet_config in table_facet_configs:
+        if isinstance(facet_config, str):
             type = "column"
-            metadata_config = {"simple": metadata_config}
+            facet_config = {"simple": facet_config}
         else:
             assert (
-                len(metadata_config.values()) == 1
+                len(facet_config.values()) == 1
             ), "Metadata config dicts should be {type: config}"
-            type, metadata_config = metadata_config.items()[0]
-            if isinstance(metadata_config, str):
-                metadata_config = {"simple": metadata_config}
+            type, facet_config = list(facet_config.items())[0]
+            if isinstance(facet_config, str):
+                facet_config = {"simple": facet_config}
         facet_configs.setdefault(type, []).append(
-            {"source": "metadata", "config": metadata_config}
+            {"source": "metadata", "config": facet_config}
         )
     qs_pairs = urllib.parse.parse_qs(request.query_string, keep_blank_values=True)
     for key, values in qs_pairs.items():
@@ -47,13 +45,12 @@ def load_facet_configs(request, table_metadata):
             elif key.startswith("_facet_"):
                 type = key[len("_facet_") :]
             for value in values:
-                # The value is the config - either JSON or not
-                if value.startswith("{"):
-                    config = json.loads(value)
-                else:
-                    config = {"simple": value}
+                # The value is the facet_config - either JSON or not
+                facet_config = (
+                    json.loads(value) if value.startswith("{") else {"simple": value}
+                )
                 facet_configs.setdefault(type, []).append(
-                    {"source": "request", "config": config}
+                    {"source": "request", "config": facet_config}
                 )
     return facet_configs
 
@@ -68,6 +65,8 @@ def register_facet_classes():
 
 class Facet:
     type = None
+    # How many rows to consider when suggesting facets:
+    suggest_consider = 1000
 
     def __init__(
         self,
@@ -77,23 +76,23 @@ class Facet:
         sql=None,
         table=None,
         params=None,
-        metadata=None,
+        table_config=None,
         row_count=None,
     ):
         assert table or sql, "Must provide either table= or sql="
         self.ds = ds
         self.request = request
         self.database = database
-        # For foreign key expansion. Can be None for e.g. canned SQL queries:
+        # For foreign key expansion. Can be None for e.g. stored SQL queries:
         self.table = table
-        self.sql = sql or "select * from [{}]".format(table)
+        self.sql = sql or f"select * from [{table}]"
         self.params = params or []
-        self.metadata = metadata
+        self.table_config = table_config
         # row_count can be None, in which case we calculate it ourselves:
         self.row_count = row_count
 
     def get_configs(self):
-        configs = load_facet_configs(self.request, self.metadata)
+        configs = load_facet_configs(self.request, self.table_config)
         return configs.get(self.type) or []
 
     def get_querystring_pairs(self):
@@ -101,6 +100,36 @@ class Facet:
         # [('_foo', 'bar'), ('_foo', '2'), ('empty', '')]
         return urllib.parse.parse_qsl(self.request.query_string, keep_blank_values=True)
 
+    def get_facet_size(self):
+        facet_size = self.ds.setting("default_facet_size")
+        max_returned_rows = self.ds.setting("max_returned_rows")
+        table_facet_size = None
+        if self.table:
+            config_facet_size = (
+                self.ds.config.get("databases", {})
+                .get(self.database, {})
+                .get("tables", {})
+                .get(self.table, {})
+                .get("facet_size")
+            )
+            if config_facet_size:
+                table_facet_size = config_facet_size
+        custom_facet_size = self.request.args.get("_facet_size")
+        if custom_facet_size:
+            if custom_facet_size == "max":
+                facet_size = max_returned_rows
+            elif custom_facet_size.isdigit():
+                facet_size = int(custom_facet_size)
+            else:
+                # Invalid value, ignore it
+                custom_facet_size = None
+        if table_facet_size and not custom_facet_size:
+            if table_facet_size == "max":
+                facet_size = max_returned_rows
+            else:
+                facet_size = table_facet_size
+        return min(facet_size, max_returned_rows)
+
     async def suggest(self):
         return []
 
@@ -114,21 +143,10 @@ class Facet:
         # Detect column names using the "limit 0" trick
         return (
             await self.ds.execute(
-                self.database, "select * from ({}) limit 0".format(sql), params or []
+                self.database, f"select * from ({sql}) limit 0", params or []
             )
         ).columns
 
-    async def get_row_count(self):
-        if self.row_count is None:
-            self.row_count = (
-                await self.ds.execute(
-                    self.database,
-                    "select count(*) from ({})".format(self.sql),
-                    self.params,
-                )
-            ).rows[0][0]
-        return self.row_count
-
 
 class ColumnFacet(Facet):
     type = "column"
@@ -136,20 +154,23 @@ class ColumnFacet(Facet):
     async def suggest(self):
         row_count = await self.get_row_count()
         columns = await self.get_columns(self.sql, self.params)
-        facet_size = self.ds.config("default_facet_size")
+        facet_size = self.get_facet_size()
         suggested_facets = []
         already_enabled = [c["config"]["simple"] for c in self.get_configs()]
         for column in columns:
             if column in already_enabled:
                 continue
             suggested_facet_sql = """
-                select {column}, count(*) as n from (
-                    {sql}
-                ) where {column} is not null
-                group by {column}
+                with limited as (select * from ({sql}) limit {suggest_consider})
+                select {column} as value, count(*) as n from limited
+                where value is not null
+                group by value
                 limit {limit}
             """.format(
-                column=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
+                column=escape_sqlite(column),
+                sql=self.sql,
+                limit=facet_size + 1,
+                suggest_consider=self.suggest_consider,
             )
             distinct_values = None
             try:
@@ -158,14 +179,12 @@ class ColumnFacet(Facet):
                     suggested_facet_sql,
                     self.params,
                     truncate=False,
-                    custom_time_limit=self.ds.config("facet_suggest_time_limit_ms"),
+                    custom_time_limit=self.ds.setting("facet_suggest_time_limit_ms"),
                 )
                 num_distinct_values = len(distinct_values)
                 if (
-                    num_distinct_values
-                    and num_distinct_values > 1
+                    1 < num_distinct_values < row_count
                     and num_distinct_values <= facet_size
-                    and num_distinct_values < row_count
                     # And at least one has n > 1
                     and any(r["n"] > 1 for r in distinct_values)
                 ):
@@ -174,7 +193,11 @@ class ColumnFacet(Facet):
                             "name": column,
                             "toggle_url": self.ds.absolute_url(
                                 self.request,
-                                path_with_added_args(self.request, {"_facet": column}),
+                                self.ds.urls.path(
+                                    path_with_added_args(
+                                        self.request, {"_facet": column}
+                                    )
+                                ),
                             ),
                         }
                     )
@@ -182,13 +205,24 @@ class ColumnFacet(Facet):
                 continue
         return suggested_facets
 
+    async def get_row_count(self):
+        if self.row_count is None:
+            self.row_count = (
+                await self.ds.execute(
+                    self.database,
+                    f"select count(*) from (select * from ({self.sql}) limit {self.suggest_consider})",
+                    self.params,
+                )
+            ).rows[0][0]
+        return self.row_count
+
     async def facet_results(self):
-        facet_results = {}
+        facet_results = []
         facets_timed_out = []
 
         qs_pairs = self.get_querystring_pairs()
 
-        facet_size = self.ds.config("default_facet_size")
+        facet_size = self.get_facet_size()
         for source_and_config in self.get_configs():
             config = source_and_config["config"]
             source = source_and_config["source"]
@@ -199,46 +233,49 @@ class ColumnFacet(Facet):
                 )
                 where {col} is not null
                 group by {col} order by count desc, value limit {limit}
-            """.format(
-                col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
-            )
+            """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1)
             try:
                 facet_rows_results = await self.ds.execute(
                     self.database,
                     facet_sql,
                     self.params,
                     truncate=False,
-                    custom_time_limit=self.ds.config("facet_time_limit_ms"),
+                    custom_time_limit=self.ds.setting("facet_time_limit_ms"),
                 )
                 facet_results_values = []
-                facet_results[column] = {
-                    "name": column,
-                    "type": self.type,
-                    "hideable": source != "metadata",
-                    "toggle_url": path_with_removed_args(
-                        self.request, {"_facet": column}
-                    ),
-                    "results": facet_results_values,
-                    "truncated": len(facet_rows_results) > facet_size,
-                }
+                facet_results.append(
+                    {
+                        "name": column,
+                        "type": self.type,
+                        "hideable": source != "metadata",
+                        "toggle_url": self.ds.urls.path(
+                            path_with_removed_args(self.request, {"_facet": column})
+                        ),
+                        "results": facet_results_values,
+                        "truncated": len(facet_rows_results) > facet_size,
+                    }
+                )
                 facet_rows = facet_rows_results.rows[:facet_size]
                 if self.table:
                     # Attempt to expand foreign keys into labels
                     values = [row["value"] for row in facet_rows]
                     expanded = await self.ds.expand_foreign_keys(
-                        self.database, self.table, column, values
+                        self.request.actor, self.database, self.table, column, values
                     )
                 else:
                     expanded = {}
                 for row in facet_rows:
-                    selected = (column, str(row["value"])) in qs_pairs
+                    column_qs = column
+                    if column.startswith("_"):
+                        column_qs = "{}__exact".format(column)
+                    selected = (column_qs, str(row["value"])) in qs_pairs
                     if selected:
                         toggle_path = path_with_removed_args(
-                            self.request, {column: str(row["value"])}
+                            self.request, {column_qs: str(row["value"])}
                         )
                     else:
                         toggle_path = path_with_added_args(
-                            self.request, {column: row["value"]}
+                            self.request, {column_qs: row["value"]}
                         )
                     facet_results_values.append(
                         {
@@ -246,7 +283,7 @@ class ColumnFacet(Facet):
                             "label": expanded.get((column, row["value"]), row["value"]),
                             "count": row["count"],
                             "toggle_url": self.ds.absolute_url(
-                                self.request, toggle_path
+                                self.request, self.ds.urls.path(toggle_path)
                             ),
                             "selected": selected,
                         }
@@ -279,10 +316,14 @@ class ArrayFacet(Facet):
                 continue
             # Is every value in this column either null or a JSON array?
             suggested_facet_sql = """
+                with limited as (select * from ({sql}) limit {suggest_consider})
                 select distinct json_type({column})
-                from ({sql})
+                from limited
+                where {column} is not null and {column} != ''
             """.format(
-                column=escape_sqlite(column), sql=self.sql
+                column=escape_sqlite(column),
+                sql=self.sql,
+                suggest_consider=self.suggest_consider,
             )
             try:
                 results = await self.ds.execute(
@@ -290,22 +331,26 @@ class ArrayFacet(Facet):
                     suggested_facet_sql,
                     self.params,
                     truncate=False,
-                    custom_time_limit=self.ds.config("facet_suggest_time_limit_ms"),
+                    custom_time_limit=self.ds.setting("facet_suggest_time_limit_ms"),
                     log_sql_errors=False,
                 )
                 types = tuple(r[0] for r in results.rows)
                 if types in (("array",), ("array", None)):
-                    # Now sanity check that first 100 arrays contain only strings
+                    # Now check that first 100 arrays contain only strings
                     first_100 = [
                         v[0]
                         for v in await self.ds.execute(
                             self.database,
-                            "select {column} from ({sql}) where {column} is not null and json_array_length({column}) > 0 limit 100".format(
-                                column=escape_sqlite(column), sql=self.sql
-                            ),
+                            (
+                                "select {column} from ({sql}) "
+                                "where {column} is not null "
+                                "and {column} != '' "
+                                "and json_array_length({column}) > 0 "
+                                "limit 100"
+                            ).format(column=escape_sqlite(column), sql=self.sql),
                             self.params,
                             truncate=False,
-                            custom_time_limit=self.ds.config(
+                            custom_time_limit=self.ds.setting(
                                 "facet_suggest_time_limit_ms"
                             ),
                             log_sql_errors=False,
@@ -320,8 +365,10 @@ class ArrayFacet(Facet):
                                 "type": "array",
                                 "toggle_url": self.ds.absolute_url(
                                     self.request,
-                                    path_with_added_args(
-                                        self.request, {"_facet_array": column}
+                                    self.ds.urls.path(
+                                        path_with_added_args(
+                                            self.request, {"_facet_array": column}
+                                        )
                                     ),
                                 ),
                             }
@@ -332,21 +379,38 @@ class ArrayFacet(Facet):
 
     async def facet_results(self):
         # self.configs should be a plain list of columns
-        facet_results = {}
+        facet_results = []
         facets_timed_out = []
 
-        facet_size = self.ds.config("default_facet_size")
+        facet_size = self.get_facet_size()
         for source_and_config in self.get_configs():
             config = source_and_config["config"]
             source = source_and_config["source"]
             column = config.get("column") or config["simple"]
+            # https://github.com/simonw/datasette/issues/448
             facet_sql = """
-                select j.value as value, count(*) as count from (
-                    {sql}
-                ) join json_each({col}) j
-                group by j.value order by count desc, value limit {limit}
+                with inner as ({sql}),
+                deduped_array_items as (
+                    select
+                        distinct j.value,
+                        inner.*
+                    from
+                        json_each([inner].{col}) j
+                        join inner
+                )
+                select
+                    value as value,
+                    count(*) as count
+                from
+                    deduped_array_items
+                group by
+                    value
+                order by
+                    count(*) desc, value limit {limit}
             """.format(
-                col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
+                col=escape_sqlite(column),
+                sql=self.sql,
+                limit=facet_size + 1,
             )
             try:
                 facet_rows_results = await self.ds.execute(
@@ -354,31 +418,35 @@ class ArrayFacet(Facet):
                     facet_sql,
                     self.params,
                     truncate=False,
-                    custom_time_limit=self.ds.config("facet_time_limit_ms"),
+                    custom_time_limit=self.ds.setting("facet_time_limit_ms"),
                 )
                 facet_results_values = []
-                facet_results[column] = {
-                    "name": column,
-                    "type": self.type,
-                    "results": facet_results_values,
-                    "hideable": source != "metadata",
-                    "toggle_url": path_with_removed_args(
-                        self.request, {"_facet_array": column}
-                    ),
-                    "truncated": len(facet_rows_results) > facet_size,
-                }
+                facet_results.append(
+                    {
+                        "name": column,
+                        "type": self.type,
+                        "results": facet_results_values,
+                        "hideable": source != "metadata",
+                        "toggle_url": self.ds.urls.path(
+                            path_with_removed_args(
+                                self.request, {"_facet_array": column}
+                            )
+                        ),
+                        "truncated": len(facet_rows_results) > facet_size,
+                    }
+                )
                 facet_rows = facet_rows_results.rows[:facet_size]
                 pairs = self.get_querystring_pairs()
                 for row in facet_rows:
                     value = str(row["value"])
-                    selected = ("{}__arraycontains".format(column), value) in pairs
+                    selected = (f"{column}__arraycontains", value) in pairs
                     if selected:
                         toggle_path = path_with_removed_args(
-                            self.request, {"{}__arraycontains".format(column): value}
+                            self.request, {f"{column}__arraycontains": value}
                         )
                     else:
                         toggle_path = path_with_added_args(
-                            self.request, {"{}__arraycontains".format(column): value}
+                            self.request, {f"{column}__arraycontains": value}
                         )
                     facet_results_values.append(
                         {
@@ -410,18 +478,16 @@ class DateFacet(Facet):
             # Does this column contain any dates in the first 100 rows?
             suggested_facet_sql = """
                 select date({column}) from (
-                    {sql}
-                ) where {column} glob "????-??-*" limit 100;
-            """.format(
-                column=escape_sqlite(column), sql=self.sql
-            )
+                    select * from ({sql}) limit 100
+                ) where {column} glob "????-??-*"
+            """.format(column=escape_sqlite(column), sql=self.sql)
             try:
                 results = await self.ds.execute(
                     self.database,
                     suggested_facet_sql,
                     self.params,
                     truncate=False,
-                    custom_time_limit=self.ds.config("facet_suggest_time_limit_ms"),
+                    custom_time_limit=self.ds.setting("facet_suggest_time_limit_ms"),
                     log_sql_errors=False,
                 )
                 values = tuple(r[0] for r in results.rows)
@@ -432,8 +498,10 @@ class DateFacet(Facet):
                             "type": "date",
                             "toggle_url": self.ds.absolute_url(
                                 self.request,
-                                path_with_added_args(
-                                    self.request, {"_facet_date": column}
+                                self.ds.urls.path(
+                                    path_with_added_args(
+                                        self.request, {"_facet_date": column}
+                                    )
                                 ),
                             ),
                         }
@@ -443,10 +511,10 @@ class DateFacet(Facet):
         return suggested_facets
 
     async def facet_results(self):
-        facet_results = {}
+        facet_results = []
         facets_timed_out = []
         args = dict(self.get_querystring_pairs())
-        facet_size = self.ds.config("default_facet_size")
+        facet_size = self.get_facet_size()
         for source_and_config in self.get_configs():
             config = source_and_config["config"]
             source = source_and_config["source"]
@@ -458,40 +526,38 @@ class DateFacet(Facet):
                 )
                 where date({col}) is not null
                 group by date({col}) order by count desc, value limit {limit}
-            """.format(
-                col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1
-            )
+            """.format(col=escape_sqlite(column), sql=self.sql, limit=facet_size + 1)
             try:
                 facet_rows_results = await self.ds.execute(
                     self.database,
                     facet_sql,
                     self.params,
                     truncate=False,
-                    custom_time_limit=self.ds.config("facet_time_limit_ms"),
+                    custom_time_limit=self.ds.setting("facet_time_limit_ms"),
                 )
                 facet_results_values = []
-                facet_results[column] = {
-                    "name": column,
-                    "type": self.type,
-                    "results": facet_results_values,
-                    "hideable": source != "metadata",
-                    "toggle_url": path_with_removed_args(
-                        self.request, {"_facet_date": column}
-                    ),
-                    "truncated": len(facet_rows_results) > facet_size,
-                }
+                facet_results.append(
+                    {
+                        "name": column,
+                        "type": self.type,
+                        "results": facet_results_values,
+                        "hideable": source != "metadata",
+                        "toggle_url": path_with_removed_args(
+                            self.request, {"_facet_date": column}
+                        ),
+                        "truncated": len(facet_rows_results) > facet_size,
+                    }
+                )
                 facet_rows = facet_rows_results.rows[:facet_size]
                 for row in facet_rows:
-                    selected = str(args.get("{}__date".format(column))) == str(
-                        row["value"]
-                    )
+                    selected = str(args.get(f"{column}__date")) == str(row["value"])
                     if selected:
                         toggle_path = path_with_removed_args(
-                            self.request, {"{}__date".format(column): str(row["value"])}
+                            self.request, {f"{column}__date": str(row["value"])}
                         )
                     else:
                         toggle_path = path_with_added_args(
-                            self.request, {"{}__date".format(column): row["value"]}
+                            self.request, {f"{column}__date": row["value"]}
                         )
                     facet_results_values.append(
                         {

datasette/filters.py

@@ -1,7 +1,173 @@
+from datasette import hookimpl
+from datasette.resources import DatabaseResource
+from datasette.views.base import DatasetteError
+from datasette.utils.asgi import BadRequest
 import json
-import numbers
+from .utils import detect_json1, escape_sqlite, path_with_removed_args
 
-from .utils import detect_json1, escape_sqlite
+
+@hookimpl(specname="filters_from_request")
+def where_filters(request, database, datasette):
+    # This one deals with ?_where=
+    async def inner():
+        where_clauses = []
+        extra_wheres_for_ui = []
+        if "_where" in request.args:
+            if not await datasette.allowed(
+                action="execute-sql",
+                resource=DatabaseResource(database=database),
+                actor=request.actor,
+            ):
+                raise DatasetteError("_where= is not allowed", status=403)
+            else:
+                where_clauses.extend(request.args.getlist("_where"))
+                extra_wheres_for_ui = [
+                    {
+                        "text": text,
+                        "remove_url": path_with_removed_args(request, {"_where": text}),
+                    }
+                    for text in request.args.getlist("_where")
+                ]
+
+        return FilterArguments(
+            where_clauses,
+            extra_context={
+                "extra_wheres_for_ui": extra_wheres_for_ui,
+            },
+        )
+
+    return inner
+
+
+@hookimpl(specname="filters_from_request")
+def search_filters(request, database, table, datasette):
+    # ?_search= and _search_colname=
+    async def inner():
+        where_clauses = []
+        params = {}
+        human_descriptions = []
+        extra_context = {}
+
+        # Figure out which fts_table to use
+        table_metadata = await datasette.table_config(database, table)
+        db = datasette.get_database(database)
+        fts_table = request.args.get("_fts_table")
+        fts_table = fts_table or table_metadata.get("fts_table")
+        fts_table = fts_table or await db.fts_table(table)
+        fts_pk = request.args.get("_fts_pk", table_metadata.get("fts_pk", "rowid"))
+        search_args = {
+            key: request.args[key]
+            for key in request.args
+            if key.startswith("_search") and key != "_searchmode"
+        }
+        search = ""
+        search_mode_raw = table_metadata.get("searchmode") == "raw"
+        # Or set search mode from the querystring
+        qs_searchmode = request.args.get("_searchmode")
+        if qs_searchmode == "escaped":
+            search_mode_raw = False
+        if qs_searchmode == "raw":
+            search_mode_raw = True
+
+        extra_context["supports_search"] = bool(fts_table)
+
+        if fts_table and search_args:
+            if "_search" in search_args:
+                # Simple ?_search=xxx
+                search = search_args["_search"]
+                where_clauses.append(
+                    "{fts_pk} in (select rowid from {fts_table} where {fts_table} match {match_clause})".format(
+                        fts_table=escape_sqlite(fts_table),
+                        fts_pk=escape_sqlite(fts_pk),
+                        match_clause=(
+                            ":search" if search_mode_raw else "escape_fts(:search)"
+                        ),
+                    )
+                )
+                human_descriptions.append(f'search matches "{search}"')
+                params["search"] = search
+                extra_context["search"] = search
+            else:
+                # More complex: search against specific columns
+                for i, (key, search_text) in enumerate(search_args.items()):
+                    search_col = key.split("_search_", 1)[1]
+                    if search_col not in await db.table_columns(fts_table):
+                        raise BadRequest("Cannot search by that column")
+
+                    where_clauses.append(
+                        "rowid in (select rowid from {fts_table} where {search_col} match {match_clause})".format(
+                            fts_table=escape_sqlite(fts_table),
+                            search_col=escape_sqlite(search_col),
+                            match_clause=(
+                                ":search_{}".format(i)
+                                if search_mode_raw
+                                else "escape_fts(:search_{})".format(i)
+                            ),
+                        )
+                    )
+                    human_descriptions.append(
+                        f'search column "{search_col}" matches "{search_text}"'
+                    )
+                    params[f"search_{i}"] = search_text
+                    extra_context["search"] = search_text
+
+        return FilterArguments(where_clauses, params, human_descriptions, extra_context)
+
+    return inner
+
+
+@hookimpl(specname="filters_from_request")
+def through_filters(request, database, table, datasette):
+    # ?_search= and _search_colname=
+    async def inner():
+        where_clauses = []
+        params = {}
+        human_descriptions = []
+        extra_context = {}
+
+        # Support for ?_through={table, column, value}
+        if "_through" in request.args:
+            for through in request.args.getlist("_through"):
+                through_data = json.loads(through)
+                through_table = through_data["table"]
+                other_column = through_data["column"]
+                value = through_data["value"]
+                db = datasette.get_database(database)
+                outgoing_foreign_keys = await db.foreign_keys_for_table(through_table)
+                try:
+                    fk_to_us = [
+                        fk for fk in outgoing_foreign_keys if fk["other_table"] == table
+                    ][0]
+                except IndexError:
+                    raise DatasetteError(
+                        "Invalid _through - could not find corresponding foreign key"
+                    )
+                param = f"p{len(params)}"
+                where_clauses.append(
+                    "{our_pk} in (select {our_column} from {through_table} where {other_column} = :{param})".format(
+                        through_table=escape_sqlite(through_table),
+                        our_pk=escape_sqlite(fk_to_us["other_column"]),
+                        our_column=escape_sqlite(fk_to_us["column"]),
+                        other_column=escape_sqlite(other_column),
+                        param=param,
+                    )
+                )
+                params[param] = value
+                human_descriptions.append(f'{through_table}.{other_column} = "{value}"')
+
+        return FilterArguments(where_clauses, params, human_descriptions, extra_context)
+
+    return inner
+
+
+class FilterArguments:
+    def __init__(
+        self, where_clauses, params=None, human_descriptions=None, extra_context=None
+    ):
+        self.where_clauses = where_clauses
+        self.params = params or {}
+        self.human_descriptions = human_descriptions or []
+        self.extra_context = extra_context or {}
 
 
 class Filter:
@@ -43,7 +209,7 @@ class TemplatedFilter(Filter):
             kwargs = {"c": column}
             converted = None
         else:
-            kwargs = {"c": column, "p": "p{}".format(param_counter), "t": table}
+            kwargs = {"c": column, "p": f"p{param_counter}", "t": table}
         return self.sql_template.format(**kwargs), converted
 
     def human_clause(self, column, value):
@@ -69,12 +235,12 @@ class InFilter(Filter):
 
     def where_clause(self, table, column, value, param_counter):
         values = self.split_value(value)
-        params = [":p{}".format(param_counter + i) for i in range(len(values))]
-        sql = "{} in ({})".format(escape_sqlite(column), ", ".join(params))
+        params = [f":p{param_counter + i}" for i in range(len(values))]
+        sql = f"{escape_sqlite(column)} in ({', '.join(params)})"
         return sql, values
 
     def human_clause(self, column, value):
-        return "{} in {}".format(column, json.dumps(self.split_value(value)))
+        return f"{column} in {json.dumps(self.split_value(value))}"
 
 
 class NotInFilter(InFilter):
@@ -83,12 +249,12 @@ class NotInFilter(InFilter):
 
     def where_clause(self, table, column, value, param_counter):
         values = self.split_value(value)
-        params = [":p{}".format(param_counter + i) for i in range(len(values))]
-        sql = "{} not in ({})".format(escape_sqlite(column), ", ".join(params))
+        params = [f":p{param_counter + i}" for i in range(len(values))]
+        sql = f"{escape_sqlite(column)} not in ({', '.join(params)})"
         return sql, values
 
     def human_clause(self, column, value):
-        return "{} not in {}".format(column, json.dumps(self.split_value(value)))
+        return f"{column} not in {json.dumps(self.split_value(value))}"
 
 
 class Filters:
@@ -114,6 +280,13 @@ class Filters:
                 '{c} contains "{v}"',
                 format="%{}%",
             ),
+            TemplatedFilter(
+                "notcontains",
+                "does not contain",
+                '"{c}" not like :{p}',
+                '{c} does not contain "{v}"',
+                format="%{}%",
+            ),
             TemplatedFilter(
                 "endswith",
                 "ends with",
@@ -137,6 +310,9 @@ class Filters:
                 "lte", "\u2264", '"{c}" <= :{p}', "{c} \u2264 {v}", numeric=True
             ),
             TemplatedFilter("like", "like", '"{c}" like :{p}', '{c} like "{v}"'),
+            TemplatedFilter(
+                "notlike", "not like", '"{c}" not like :{p}', '{c} not like "{v}"'
+            ),
             TemplatedFilter("glob", "glob", '"{c}" glob :{p}', '{c} glob "{v}"'),
             InFilter(),
             NotInFilter(),
@@ -146,12 +322,15 @@ class Filters:
                 TemplatedFilter(
                     "arraycontains",
                     "array contains",
-                    """rowid in (
-            select {t}.rowid from {t}, json_each({t}.{c}) j
-            where j.value = :{p}
-        )""",
+                    """:{p} in (select value from json_each([{t}].[{c}]))""",
                     '{c} contains "{v}"',
-                )
+                ),
+                TemplatedFilter(
+                    "arraynotcontains",
+                    "array does not contain",
+                    """:{p} not in (select value from json_each([{t}].[{c}]))""",
+                    '{c} does not contain "{v}"',
+                ),
             ]
             if detect_json1()
             else []
@@ -188,13 +367,11 @@ class Filters:
     )
     _filters_by_key = {f.key: f for f in _filters}
 
-    def __init__(self, pairs, units={}, ureg=None):
+    def __init__(self, pairs):
         self.pairs = pairs
-        self.units = units
-        self.ureg = ureg
 
     def lookups(self):
-        "Yields (lookup, display, no_argument) pairs"
+        """Yields (lookup, display, no_argument) pairs"""
         for filter in self._filters:
             yield filter.key, filter.display, filter.no_argument
 
@@ -216,10 +393,10 @@ class Filters:
         s = " and ".join(and_bits)
         if not s:
             return ""
-        return "where {}".format(s)
+        return f"where {s}"
 
     def selections(self):
-        "Yields (column, lookup, value) tuples"
+        """Yields (column, lookup, value) tuples"""
         for key, value in self.pairs:
             if "__" in key:
                 column, lookup = key.rsplit("__", 1)
@@ -231,20 +408,6 @@ class Filters:
     def has_selections(self):
         return bool(self.pairs)
 
-    def convert_unit(self, column, value):
-        "If the user has provided a unit in the query, convert it into the column unit, if present."
-        if column not in self.units:
-            return value
-
-        # Try to interpret the value as a unit
-        value = self.ureg(value)
-        if isinstance(value, numbers.Number):
-            # It's just a bare number, assume it's the column unit
-            return value
-
-        column_unit = self.ureg(self.units[column])
-        return value.to(column_unit).magnitude
-
     def build_where_clauses(self, table):
         sql_bits = []
         params = {}
@@ -252,15 +415,13 @@ class Filters:
         for column, lookup, value in self.selections():
             filter = self._filters_by_key.get(lookup, None)
             if filter:
-                sql_bit, param = filter.where_clause(
-                    table, column, self.convert_unit(column, value), i
-                )
+                sql_bit, param = filter.where_clause(table, column, value, i)
                 sql_bits.append(sql_bit)
                 if param is not None:
                     if not isinstance(param, list):
                         param = [param]
                     for individual_param in param:
-                        param_id = "p{}".format(i)
+                        param_id = f"p{i}"
                         params[param_id] = individual_param
                         i += 1
         return sql_bits, params

datasette/fixtures.py

@@ -0,0 +1,415 @@
+from datasette.utils.sqlite import sqlite3
+from datasette.utils import documented
+import itertools
+import random
+import string
+
+__all__ = [
+    "EXTRA_DATABASE_SQL",
+    "TABLES",
+    "TABLE_PARAMETERIZED_SQL",
+    "generate_compound_rows",
+    "generate_sortable_rows",
+    "populate_extra_database",
+    "populate_fixture_database",
+    "write_extra_database",
+    "write_fixture_database",
+]
+
+
+def generate_compound_rows(num):
+    """Generate rows for the compound_three_primary_keys fixture table."""
+    for a, b, c in itertools.islice(
+        itertools.product(string.ascii_lowercase, repeat=3), num
+    ):
+        yield a, b, c, f"{a}-{b}-{c}"
+
+
+def generate_sortable_rows(num):
+    """Generate rows for the sortable fixture table."""
+    rand = random.Random(42)
+    for a, b in itertools.islice(
+        itertools.product(string.ascii_lowercase, repeat=2), num
+    ):
+        yield {
+            "pk1": a,
+            "pk2": b,
+            "content": f"{a}-{b}",
+            "sortable": rand.randint(-100, 100),
+            "sortable_with_nulls": rand.choice([None, rand.random(), rand.random()]),
+            "sortable_with_nulls_2": rand.choice([None, rand.random(), rand.random()]),
+            "text": rand.choice(["$null", "$blah"]),
+        }
+
+
+TABLES = (
+    """
+CREATE TABLE simple_primary_key (
+  id integer primary key,
+  content text
+);
+
+CREATE TABLE primary_key_multiple_columns (
+  id varchar(30) primary key,
+  content text,
+  content2 text
+);
+
+CREATE TABLE primary_key_multiple_columns_explicit_label (
+  id varchar(30) primary key,
+  content text,
+  content2 text
+);
+
+CREATE TABLE compound_primary_key (
+  pk1 varchar(30),
+  pk2 varchar(30),
+  content text,
+  PRIMARY KEY (pk1, pk2)
+);
+
+INSERT INTO compound_primary_key VALUES ('a', 'b', 'c');
+INSERT INTO compound_primary_key VALUES ('a/b', '.c-d', 'c');
+INSERT INTO compound_primary_key VALUES ('d', 'e', 'RENDER_CELL_DEMO');
+
+CREATE TABLE compound_three_primary_keys (
+  pk1 varchar(30),
+  pk2 varchar(30),
+  pk3 varchar(30),
+  content text,
+  PRIMARY KEY (pk1, pk2, pk3)
+);
+CREATE INDEX idx_compound_three_primary_keys_content ON compound_three_primary_keys(content);
+
+CREATE TABLE foreign_key_references (
+  pk varchar(30) primary key,
+  foreign_key_with_label integer,
+  foreign_key_with_blank_label integer,
+  foreign_key_with_no_label varchar(30),
+  foreign_key_compound_pk1 varchar(30),
+  foreign_key_compound_pk2 varchar(30),
+  FOREIGN KEY (foreign_key_with_label) REFERENCES simple_primary_key(id),
+  FOREIGN KEY (foreign_key_with_blank_label) REFERENCES simple_primary_key(id),
+  FOREIGN KEY (foreign_key_with_no_label) REFERENCES primary_key_multiple_columns(id)
+  FOREIGN KEY (foreign_key_compound_pk1, foreign_key_compound_pk2) REFERENCES compound_primary_key(pk1, pk2)
+);
+
+CREATE TABLE sortable (
+  pk1 varchar(30),
+  pk2 varchar(30),
+  content text,
+  sortable integer,
+  sortable_with_nulls real,
+  sortable_with_nulls_2 real,
+  text text,
+  PRIMARY KEY (pk1, pk2)
+);
+
+CREATE TABLE no_primary_key (
+  content text,
+  a text,
+  b text,
+  c text
+);
+
+CREATE TABLE [123_starts_with_digits] (
+  content text
+);
+
+CREATE VIEW paginated_view AS
+    SELECT
+        content,
+        '- ' || content || ' -' AS content_extra
+    FROM no_primary_key;
+
+CREATE TABLE "Table With Space In Name" (
+  pk varchar(30) primary key,
+  content text
+);
+
+CREATE TABLE "table/with/slashes.csv" (
+  pk varchar(30) primary key,
+  content text
+);
+
+CREATE TABLE "complex_foreign_keys" (
+  pk varchar(30) primary key,
+  f1 integer,
+  f2 integer,
+  f3 integer,
+  FOREIGN KEY ("f1") REFERENCES [simple_primary_key](id),
+  FOREIGN KEY ("f2") REFERENCES [simple_primary_key](id),
+  FOREIGN KEY ("f3") REFERENCES [simple_primary_key](id)
+);
+
+CREATE TABLE "custom_foreign_key_label" (
+  pk varchar(30) primary key,
+  foreign_key_with_custom_label text,
+  FOREIGN KEY ("foreign_key_with_custom_label") REFERENCES [primary_key_multiple_columns_explicit_label](id)
+);
+
+CREATE TABLE tags (
+    tag TEXT PRIMARY KEY
+);
+
+CREATE TABLE searchable (
+  pk integer primary key,
+  text1 text,
+  text2 text,
+  [name with . and spaces] text
+);
+
+CREATE TABLE searchable_tags (
+    searchable_id integer,
+    tag text,
+    PRIMARY KEY (searchable_id, tag),
+    FOREIGN KEY (searchable_id) REFERENCES searchable(pk),
+    FOREIGN KEY (tag) REFERENCES tags(tag)
+);
+
+INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog', 'panther');
+INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel', 'puma');
+
+INSERT INTO tags VALUES ("canine");
+INSERT INTO tags VALUES ("feline");
+
+INSERT INTO searchable_tags (searchable_id, tag) VALUES
+    (1, "feline"),
+    (2, "canine")
+;
+
+CREATE VIRTUAL TABLE "searchable_fts"
+    USING FTS5 (text1, text2, [name with . and spaces], content="searchable", content_rowid="pk");
+INSERT INTO "searchable_fts" (searchable_fts) VALUES ('rebuild');
+
+CREATE TABLE [select] (
+  [group] text,
+  [having] text,
+  [and] text,
+  [json] text
+);
+INSERT INTO [select] VALUES ('group', 'having', 'and',
+    '{"href": "http://example.com/", "label":"Example"}'
+);
+
+CREATE TABLE infinity (
+    value REAL
+);
+INSERT INTO infinity VALUES
+    (1e999),
+    (-1e999),
+    (1.5)
+;
+
+CREATE TABLE facet_cities (
+    id integer primary key,
+    name text
+);
+INSERT INTO facet_cities (id, name) VALUES
+    (1, 'San Francisco'),
+    (2, 'Los Angeles'),
+    (3, 'Detroit'),
+    (4, 'Memnonia')
+;
+
+CREATE TABLE facetable (
+    pk integer primary key,
+    created text,
+    planet_int integer,
+    on_earth integer,
+    state text,
+    _city_id integer,
+    _neighborhood text,
+    tags text,
+    complex_array text,
+    distinct_some_null,
+    n text,
+    FOREIGN KEY ("_city_id") REFERENCES [facet_cities](id)
+);
+INSERT INTO facetable
+    (created, planet_int, on_earth, state, _city_id, _neighborhood, tags, complex_array, distinct_some_null, n)
+VALUES
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Mission', '["tag1", "tag2"]', '[{"foo": "bar"}]', 'one', 'n1'),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Dogpatch', '["tag1", "tag3"]', '[]', 'two', 'n2'),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'SOMA', '[]', '[]', null, null),
+    ("2019-01-14 08:00:00", 1, 1, 'CA', 1, 'Tenderloin', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Bernal Heights', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 1, 'Hayes Valley', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Hollywood', '[]', '[]', null, null),
+    ("2019-01-15 08:00:00", 1, 1, 'CA', 2, 'Downtown', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Los Feliz', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'CA', 2, 'Koreatown', '[]', '[]', null, null),
+    ("2019-01-16 08:00:00", 1, 1, 'MI', 3, 'Downtown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Greektown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Corktown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 1, 1, 'MI', 3, 'Mexicantown', '[]', '[]', null, null),
+    ("2019-01-17 08:00:00", 2, 0, 'MC', 4, 'Arcadia Planitia', '[]', '[]', null, null)
+;
+
+CREATE TABLE binary_data (
+    data BLOB
+);
+
+-- Many 2 Many demo: roadside attractions!
+
+CREATE TABLE roadside_attractions (
+    pk integer primary key,
+    name text,
+    address text,
+    url text,
+    latitude real,
+    longitude real
+);
+INSERT INTO roadside_attractions VALUES (
+    1, "The Mystery Spot", "465 Mystery Spot Road, Santa Cruz, CA 95065", "https://www.mysteryspot.com/",
+    37.0167, -122.0024
+);
+INSERT INTO roadside_attractions VALUES (
+    2, "Winchester Mystery House", "525 South Winchester Boulevard, San Jose, CA 95128", "https://winchestermysteryhouse.com/",
+    37.3184, -121.9511
+);
+INSERT INTO roadside_attractions VALUES (
+    3, "Burlingame Museum of PEZ Memorabilia", "214 California Drive, Burlingame, CA 94010", null,
+    37.5793, -122.3442
+);
+INSERT INTO roadside_attractions VALUES (
+    4, "Bigfoot Discovery Museum", "5497 Highway 9, Felton, CA 95018", "https://www.bigfootdiscoveryproject.com/",
+    37.0414, -122.0725
+);
+
+CREATE TABLE attraction_characteristic (
+    pk integer primary key,
+    name text
+);
+INSERT INTO attraction_characteristic VALUES (
+    1, "Museum"
+);
+INSERT INTO attraction_characteristic VALUES (
+    2, "Paranormal"
+);
+
+CREATE TABLE roadside_attraction_characteristics (
+    attraction_id INTEGER REFERENCES roadside_attractions(pk),
+    characteristic_id INTEGER REFERENCES attraction_characteristic(pk)
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    1, 2
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    2, 2
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    4, 2
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    3, 1
+);
+INSERT INTO roadside_attraction_characteristics VALUES (
+    4, 1
+);
+
+INSERT INTO simple_primary_key VALUES (1, 'hello');
+INSERT INTO simple_primary_key VALUES (2, 'world');
+INSERT INTO simple_primary_key VALUES (3, '');
+INSERT INTO simple_primary_key VALUES (4, 'RENDER_CELL_DEMO');
+INSERT INTO simple_primary_key VALUES (5, 'RENDER_CELL_ASYNC');
+
+INSERT INTO primary_key_multiple_columns VALUES (1, 'hey', 'world');
+INSERT INTO primary_key_multiple_columns_explicit_label VALUES (1, 'hey', 'world2');
+
+INSERT INTO foreign_key_references VALUES (1, 1, 3, 1, 'a', 'b');
+INSERT INTO foreign_key_references VALUES (2, null, null, null, null, null);
+
+INSERT INTO complex_foreign_keys VALUES (1, 1, 2, 1);
+INSERT INTO custom_foreign_key_label VALUES (1, 1);
+
+INSERT INTO [table/with/slashes.csv] VALUES (3, 'hey');
+
+CREATE VIEW simple_view AS
+    SELECT content, upper(content) AS upper_content FROM simple_primary_key;
+
+CREATE VIEW searchable_view AS
+    SELECT * from searchable;
+
+CREATE VIEW searchable_view_configured_by_metadata AS
+    SELECT * from searchable;
+
+"""
+    + "\n".join(
+        [
+            'INSERT INTO no_primary_key VALUES ({i}, "a{i}", "b{i}", "c{i}");'.format(
+                i=i + 1
+            )
+            for i in range(201)
+        ]
+    )
+    + '\nINSERT INTO no_primary_key VALUES ("RENDER_CELL_DEMO", "a202", "b202", "c202");\n'
+    + "\n".join(
+        [
+            'INSERT INTO compound_three_primary_keys VALUES ("{a}", "{b}", "{c}", "{content}");'.format(
+                a=a, b=b, c=c, content=content
+            )
+            for a, b, c, content in generate_compound_rows(1001)
+        ]
+    )
+    + "\n".join(["""INSERT INTO sortable VALUES (
+        "{pk1}", "{pk2}", "{content}", {sortable},
+        {sortable_with_nulls}, {sortable_with_nulls_2}, "{text}");
+    """.format(**row).replace("None", "null") for row in generate_sortable_rows(201)])
+)
+
+TABLE_PARAMETERIZED_SQL = [
+    ("insert into binary_data (data) values (?);", [b"\x15\x1c\x02\xc7\xad\x05\xfe"]),
+    ("insert into binary_data (data) values (?);", [b"\x15\x1c\x03\xc7\xad\x05\xfe"]),
+    ("insert into binary_data (data) values (null);", []),
+]
+
+EXTRA_DATABASE_SQL = """
+CREATE TABLE searchable (
+  pk integer primary key,
+  text1 text,
+  text2 text
+);
+
+CREATE VIEW searchable_view AS SELECT * FROM searchable;
+
+INSERT INTO searchable VALUES (1, 'barry cat', 'terry dog');
+INSERT INTO searchable VALUES (2, 'terry dog', 'sara weasel');
+
+CREATE VIRTUAL TABLE "searchable_fts"
+    USING FTS3 (text1, text2, content="searchable");
+INSERT INTO "searchable_fts" (rowid, text1, text2)
+    SELECT rowid, text1, text2 FROM searchable;
+"""
+
+
+@documented(label="datasette_fixtures_populate_fixture_database")
+def populate_fixture_database(conn):
+    """Populate a SQLite connection with Datasette's test fixture tables."""
+    conn.executescript(TABLES)
+    for sql, params in TABLE_PARAMETERIZED_SQL:
+        with conn:
+            conn.execute(sql, params)
+
+
+def populate_extra_database(conn):
+    """Populate a SQLite connection with the extra database used in tests."""
+    conn.executescript(EXTRA_DATABASE_SQL)
+
+
+def write_fixture_database(db_filename):
+    """Write Datasette's test fixture tables to a SQLite database file."""
+    conn = sqlite3.connect(db_filename)
+    try:
+        populate_fixture_database(conn)
+    finally:
+        conn.close()
+
+
+def write_extra_database(db_filename):
+    """Write the extra test database tables to a SQLite database file."""
+    conn = sqlite3.connect(db_filename)
+    try:
+        populate_extra_database(conn)
+    finally:
+        conn.close()

datasette/forbidden.py

@@ -0,0 +1,19 @@
+from datasette import hookimpl, Response
+
+
+@hookimpl(trylast=True)
+def forbidden(datasette, request, message):
+    async def inner():
+        return Response.html(
+            await datasette.render_template(
+                "error.html",
+                {
+                    "title": "Forbidden",
+                    "error": message,
+                },
+                request=request,
+            ),
+            status=403,
+        )
+
+    return inner

datasette/handle_exception.py

@@ -0,0 +1,77 @@
+from datasette import hookimpl, Response
+from .utils import add_cors_headers
+from .utils.asgi import (
+    Base400,
+)
+from .views.base import DatasetteError
+from markupsafe import Markup
+import traceback
+
+try:
+    import ipdb as pdb
+except ImportError:
+    import pdb
+
+try:
+    import rich
+except ImportError:
+    rich = None
+
+
+@hookimpl(trylast=True)
+def handle_exception(datasette, request, exception):
+    async def inner():
+        if datasette.pdb:
+            pdb.post_mortem(exception.__traceback__)
+
+        if rich is not None:
+            rich.get_console().print_exception(show_locals=True)
+
+        title = None
+        if isinstance(exception, Base400):
+            status = exception.status
+            info = {}
+            message = exception.args[0]
+        elif isinstance(exception, DatasetteError):
+            status = exception.status
+            info = exception.error_dict
+            message = exception.message
+            if exception.message_is_html:
+                message = Markup(message)
+            title = exception.title
+        else:
+            status = 500
+            info = {}
+            message = str(exception)
+            traceback.print_exc()
+        templates = [f"{status}.html", "error.html"]
+        info.update(
+            {
+                "ok": False,
+                "error": message,
+                "status": status,
+                "title": title,
+            }
+        )
+        headers = {}
+        if datasette.cors:
+            add_cors_headers(headers)
+        if request.path.split("?")[0].endswith(".json"):
+            return Response.json(info, status=status, headers=headers)
+        else:
+            environment = datasette.get_jinja_environment(request)
+            template = environment.select_template(templates)
+            return Response.html(
+                await template.render_async(
+                    dict(
+                        info,
+                        urls=datasette.urls,
+                        app_css_hash=datasette.app_css_hash(),
+                        menu_links=lambda: [],
+                    )
+                ),
+                status=status,
+                headers=headers,
+            )
+
+    return inner

datasette/hookspecs.py

@@ -5,56 +5,261 @@ hookspec = HookspecMarker("datasette")
 hookimpl = HookimplMarker("datasette")
 
 
+@hookspec
+def startup(datasette):
+    """Fires directly after Datasette first starts running"""
+
+
 @hookspec
 def asgi_wrapper(datasette):
-    "Returns an ASGI middleware callable to wrap our ASGI application with"
+    """Returns an ASGI middleware callable to wrap our ASGI application with"""
 
 
 @hookspec
 def prepare_connection(conn, database, datasette):
-    "Modify SQLite connection in some way e.g. register custom SQL functions"
+    """Modify SQLite connection in some way e.g. register custom SQL functions"""
 
 
 @hookspec
-def prepare_jinja2_environment(env):
-    "Modify Jinja2 template environment e.g. register custom template tags"
+def prepare_jinja2_environment(env, datasette):
+    """Modify Jinja2 template environment e.g. register custom template tags"""
 
 
 @hookspec
-def extra_css_urls(template, database, table, datasette):
-    "Extra CSS URLs added by this plugin"
+def extra_css_urls(template, database, table, columns, view_name, request, datasette):
+    """Extra CSS URLs added by this plugin"""
 
 
 @hookspec
-def extra_js_urls(template, database, table, datasette):
-    "Extra JavaScript URLs added by this plugin"
+def extra_js_urls(template, database, table, columns, view_name, request, datasette):
+    """Extra JavaScript URLs added by this plugin"""
 
 
 @hookspec
-def extra_body_script(template, database, table, view_name, datasette):
-    "Extra JavaScript code to be included in <script> at bottom of body"
+def extra_body_script(
+    template, database, table, columns, view_name, request, datasette
+):
+    """Extra JavaScript code to be included in <script> at bottom of body"""
 
 
 @hookspec
-def extra_template_vars(template, database, table, view_name, request, datasette):
-    "Extra template variables to be made available to the template - can return dict or callable or awaitable"
+def extra_template_vars(
+    template, database, table, columns, view_name, request, datasette
+):
+    """Extra template variables to be made available to the template - can return dict or callable or awaitable"""
 
 
 @hookspec
 def publish_subcommand(publish):
-    "Subcommands for 'datasette publish'"
+    """Subcommands for 'datasette publish'"""
 
 
-@hookspec(firstresult=True)
-def render_cell(value, column, table, database, datasette):
-    "Customize rendering of HTML table cell values"
+@hookspec
+def render_cell(
+    row,
+    value,
+    column,
+    table,
+    pks,
+    database,
+    datasette,
+    request,
+    column_type,
+):
+    """Customize rendering of HTML table cell values"""
 
 
 @hookspec
 def register_output_renderer(datasette):
-    "Register a renderer to output data in a different format"
+    """Register a renderer to output data in a different format"""
 
 
 @hookspec
 def register_facet_classes():
-    "Register Facet subclasses"
+    """Register Facet subclasses"""
+
+
+@hookspec
+def register_actions(datasette):
+    """Register actions: returns a list of datasette.permission.Action objects"""
+
+
+@hookspec
+def register_column_types(datasette):
+    """Return a list of ColumnType subclasses"""
+
+
+@hookspec
+def register_routes(datasette):
+    """Register URL routes: return a list of (regex, view_function) pairs"""
+
+
+@hookspec
+def register_commands(cli):
+    """Register additional CLI commands, e.g. 'datasette mycommand ...'"""
+
+
+@hookspec
+def actor_from_request(datasette, request):
+    """Return an actor dictionary based on the incoming request"""
+
+
+@hookspec(firstresult=True)
+def actors_from_ids(datasette, actor_ids):
+    """Returns a dictionary mapping those IDs to actor dictionaries"""
+
+
+@hookspec
+def jinja2_environment_from_request(datasette, request, env):
+    """Return a Jinja2 environment based on the incoming request"""
+
+
+@hookspec
+def filters_from_request(request, database, table, datasette):
+    """
+    Return datasette.filters.FilterArguments(
+        where_clauses=[str, str, str],
+        params={},
+        human_descriptions=[str, str, str],
+        extra_context={}
+    ) based on the request"""
+
+
+@hookspec
+def permission_resources_sql(datasette, actor, action):
+    """Return SQL query fragments for permission checks on resources.
+
+    Returns None, a PermissionSQL object, or a list of PermissionSQL objects.
+    Each PermissionSQL contains SQL that should return rows with columns:
+    parent (str|None), child (str|None), allow (int), reason (str).
+
+    Used to efficiently check permissions across multiple resources at once.
+    """
+
+
+@hookspec
+def register_magic_parameters(datasette):
+    """Return a list of (name, function) magic parameter functions"""
+
+
+@hookspec
+def forbidden(datasette, request, message):
+    """Custom response for a 403 forbidden error"""
+
+
+@hookspec
+def menu_links(datasette, actor, request):
+    """Links for the navigation menu"""
+
+
+@hookspec
+def jump_items_sql(datasette, actor, request):
+    """SQL fragments for extra items in the jump menu"""
+
+
+@hookspec
+def row_actions(datasette, actor, request, database, table, row):
+    """Links for the row actions menu"""
+
+
+@hookspec
+def table_actions(datasette, actor, database, table, request):
+    """Links for the table actions menu"""
+
+
+@hookspec
+def view_actions(datasette, actor, database, view, request):
+    """Links for the view actions menu"""
+
+
+@hookspec
+def query_actions(datasette, actor, database, query_name, request, sql, params):
+    """Links for the query and stored query actions menu"""
+
+
+@hookspec
+def database_actions(datasette, actor, database, request):
+    """Links for the database actions menu"""
+
+
+@hookspec
+def homepage_actions(datasette, actor, request):
+    """Links for the homepage actions menu"""
+
+
+@hookspec
+def handle_exception(datasette, request, exception):
+    """Handle an uncaught exception. Can return a Response or None."""
+
+
+@hookspec
+def track_event(datasette, event):
+    """Respond to an event tracked by Datasette"""
+
+
+@hookspec
+def register_events(datasette):
+    """Return a list of Event subclasses to use with track_event()"""
+
+
+@hookspec
+def top_homepage(datasette, request):
+    """HTML to include at the top of the homepage"""
+
+
+@hookspec
+def top_database(datasette, request, database):
+    """HTML to include at the top of the database page"""
+
+
+@hookspec
+def top_table(datasette, request, database, table):
+    """HTML to include at the top of the table page"""
+
+
+@hookspec
+def top_row(datasette, request, database, table, row):
+    """HTML to include at the top of the row page"""
+
+
+@hookspec
+def top_query(datasette, request, database, sql):
+    """HTML to include at the top of the query results page"""
+
+
+@hookspec
+def top_stored_query(datasette, request, database, query_name):
+    """HTML to include at the top of the stored query page"""
+
+
+@hookspec
+def register_token_handler(datasette):
+    """Return a TokenHandler instance for token creation and verification"""
+
+
+@hookspec
+def write_wrapper(datasette, database, request, transaction):
+    """Called when a write function is about to execute.
+
+    Return a generator function that accepts a ``conn`` argument and
+    optionally a ``track_event`` argument.  The generator should
+    ``yield`` exactly once: code before the ``yield`` runs before
+    the write, code after the ``yield`` runs after the write
+    completes. The result of the write is sent back through the
+    ``yield``, so you can capture it with ``result = yield``.
+
+    If your generator accepts ``track_event``, you can call
+    ``track_event(event)`` to queue an event that will be dispatched
+    via ``datasette.track_event()`` after the write commits
+    successfully.  Events are discarded if the write raises an
+    exception.
+
+    If the write raises an exception, it is thrown into the generator
+    so you can handle it with a try/except around the ``yield``.
+
+    ``request`` may be ``None`` for writes not originating from an
+    HTTP request.  ``transaction`` is ``True`` if the write will
+    be wrapped in a transaction.
+
+    Return ``None`` to skip wrapping.
+    """

datasette/inspect.py

@@ -10,12 +10,11 @@ from .utils import (
     sqlite3,
 )
 
-
 HASH_BLOCK_SIZE = 1024 * 1024
 
 
 def inspect_hash(path):
-    " Calculate the hash of a database, efficiently. "
+    """Calculate the hash of a database, efficiently."""
     m = hashlib.sha256()
     with path.open("rb") as fp:
         while True:
@@ -28,14 +27,14 @@ def inspect_hash(path):
 
 
 def inspect_views(conn):
-    " List views in a database. "
+    """List views in a database."""
     return [
         v[0] for v in conn.execute('select name from sqlite_master where type = "view"')
     ]
 
 
 def inspect_tables(conn, database_metadata):
-    " List tables and their row counts, excluding uninteresting tables. "
+    """List tables and their row counts, excluding uninteresting tables."""
     tables = {}
     table_names = [
         r["name"]
@@ -47,7 +46,7 @@ def inspect_tables(conn, database_metadata):
 
         try:
             count = conn.execute(
-                "select count(*) from {}".format(escape_sqlite(table))
+                f"select count(*) from {escape_sqlite(table)}"
             ).fetchone()[0]
         except sqlite3.OperationalError:
             # This can happen when running against a FTS virtual table
@@ -70,16 +69,11 @@ def inspect_tables(conn, database_metadata):
         tables[table]["foreign_keys"] = info
 
     # Mark tables 'hidden' if they relate to FTS virtual tables
-    hidden_tables = [
-        r["name"]
-        for r in conn.execute(
-            """
+    hidden_tables = [r["name"] for r in conn.execute("""
                 select name from sqlite_master
                 where rootpage = 0
                 and sql like '%VIRTUAL TABLE%USING FTS%'
-            """
-        )
-    ]
+            """)]
 
     if detect_spatialite(conn):
         # Also hide Spatialite internal tables
@@ -94,14 +88,11 @@ def inspect_tables(conn, database_metadata):
             "views_geometry_columns",
             "virts_geometry_columns",
         ] + [
-            r["name"]
-            for r in conn.execute(
-                """
+            r["name"] for r in conn.execute("""
                     select name from sqlite_master
                     where name like "idx_%"
                     and type = "table"
-                """
-            )
+                """)
         ]
 
     for t in tables.keys():

datasette/jump.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class JumpSQL:
+    sql: str
+    params: dict[str, Any] | None = None
+    database: str | None = None
+
+    @classmethod
+    def menu_item(
+        cls,
+        *,
+        label: str,
+        url: str,
+        description: str = "Menu item",
+        search_text: str | None = None,
+        display_name: str | None = None,
+        item_type: str = "menu",
+    ) -> "JumpSQL":
+        if search_text is None:
+            search_text = " ".join(
+                text for text in (label, display_name, description) if text is not None
+            )
+        return cls(
+            sql="""
+            SELECT
+                :type AS type,
+                :label AS label,
+                :description AS description,
+                :url AS url,
+                :search_text AS search_text,
+                :display_name AS display_name
+            """,
+            params={
+                "type": item_type,
+                "label": label,
+                "description": description,
+                "url": url,
+                "search_text": search_text,
+                "display_name": display_name,
+            },
+        )
+
+
+_PARAM_RE = re.compile(r"(?<!:):([A-Za-z_][A-Za-z0-9_]*)")
+
+
+def namespace_sql_params(sql: str, params: dict[str, Any], prefix: str):
+    """Rename named SQL parameters so UNION query parameters cannot collide."""
+    if not params:
+        return sql, {}
+
+    renamed = {key: f"{prefix}_{key}" for key in params}
+
+    def replace(match):
+        key = match.group(1)
+        if key not in renamed:
+            return match.group(0)
+        return f":{renamed[key]}"
+
+    return _PARAM_RE.sub(replace, sql), {
+        renamed[key]: value for key, value in params.items()
+    }

datasette/permissions.py

@@ -0,0 +1,219 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import Any, NamedTuple
+import contextvars
+
+# Context variable to track when permission checks should be skipped
+_skip_permission_checks = contextvars.ContextVar(
+    "skip_permission_checks", default=False
+)
+
+
+class SkipPermissions:
+    """Context manager to temporarily skip permission checks.
+
+    This is not a stable API and may change in future releases.
+
+    Usage:
+        with SkipPermissions():
+            # Permission checks are skipped within this block
+            response = await datasette.client.get("/protected")
+    """
+
+    def __enter__(self):
+        self.token = _skip_permission_checks.set(True)
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        _skip_permission_checks.reset(self.token)
+        return False
+
+
+class Resource(ABC):
+    """
+    Base class for all resource types.
+
+    Each subclass represents a type of resource (e.g., TableResource, DatabaseResource).
+    The class itself carries metadata about the resource type.
+    Instances represent specific resources.
+    """
+
+    # Class-level metadata (subclasses must define these)
+    name: str = None  # e.g., "table", "database", "model"
+    parent_class: type["Resource"] | None = None  # e.g., DatabaseResource for tables
+
+    # Instance-level optional extra attributes
+    reasons: list[str] | None = None
+    include_reasons: bool | None = None
+
+    def __init__(self, parent: str | None = None, child: str | None = None):
+        """
+        Create a resource instance.
+
+        Args:
+            parent: The parent identifier (meaning depends on resource type)
+            child: The child identifier (meaning depends on resource type)
+        """
+        self.parent = parent
+        self.child = child
+        self._private = None  # Sentinel to track if private was set
+
+    def __str__(self) -> str:
+        return "/".join(
+            str(part) for part in (self.parent, self.child) if part is not None
+        )
+
+    def __repr__(self) -> str:
+        return "{}(parent={!r}, child={!r})".format(
+            self.__class__.__name__, self.parent, self.child
+        )
+
+    @property
+    def private(self) -> bool:
+        """
+        Whether this resource is private (accessible to actor but not anonymous).
+
+        This property is only available on Resource objects returned from
+        allowed_resources() when include_is_private=True is used.
+
+        Raises:
+            AttributeError: If accessed without calling include_is_private=True
+        """
+        if self._private is None:
+            raise AttributeError(
+                "The 'private' attribute is only available when using "
+                "allowed_resources(..., include_is_private=True)"
+            )
+        return self._private
+
+    @private.setter
+    def private(self, value: bool):
+        self._private = value
+
+    @classmethod
+    def __init_subclass__(cls):
+        """
+        Validate resource hierarchy doesn't exceed 2 levels.
+
+        Raises:
+            ValueError: If this resource would create a 3-level hierarchy
+        """
+        super().__init_subclass__()
+
+        if cls.parent_class is None:
+            return  # Top of hierarchy, nothing to validate
+
+        # Check if our parent has a parent - that would create 3 levels
+        if cls.parent_class.parent_class is not None:
+            # We have a parent, and that parent has a parent
+            # This creates a 3-level hierarchy, which is not allowed
+            raise ValueError(
+                f"Resource {cls.__name__} creates a 3-level hierarchy: "
+                f"{cls.parent_class.parent_class.__name__} -> {cls.parent_class.__name__} -> {cls.__name__}. "
+                f"Maximum 2 levels allowed (parent -> child)."
+            )
+
+    @classmethod
+    @abstractmethod
+    async def resources_sql(cls, datasette, actor=None) -> str:
+        """
+        Return SQL query that returns all resources of this type.
+
+        Must return two columns: parent, child
+        """
+        pass
+
+
+class AllowedResource(NamedTuple):
+    """A resource with the reason it was allowed (for debugging)."""
+
+    resource: Resource
+    reason: str
+
+
+@dataclass(frozen=True, kw_only=True)
+class Action:
+    name: str
+    description: str | None
+    abbr: str | None = None
+    resource_class: type[Resource] | None = None
+    also_requires: str | None = None  # Optional action name that must also be allowed
+
+    @property
+    def takes_parent(self) -> bool:
+        """
+        Whether this action requires a parent identifier when instantiating its resource.
+
+        Returns False for global-only actions (no resource_class).
+        Returns True for all actions with a resource_class (all resources require a parent identifier).
+        """
+        return self.resource_class is not None
+
+    @property
+    def takes_child(self) -> bool:
+        """
+        Whether this action requires a child identifier when instantiating its resource.
+
+        Returns False for global actions (no resource_class).
+        Returns False for parent-level resources (DatabaseResource - parent_class is None).
+        Returns True for child-level resources (TableResource, QueryResource - have a parent_class).
+        """
+        if self.resource_class is None:
+            return False
+        return self.resource_class.parent_class is not None
+
+
+_reason_id = 1
+
+
+@dataclass
+class PermissionSQL:
+    """
+    A plugin contributes SQL that yields:
+      parent TEXT NULL,
+      child  TEXT NULL,
+      allow  INTEGER,    -- 1 allow, 0 deny
+      reason TEXT
+
+    For restriction-only plugins, sql can be None and only restriction_sql is provided.
+    """
+
+    sql: str | None = (
+        None  # SQL that SELECTs the 4 columns above (can be None for restriction-only)
+    )
+    params: dict[str, Any] | None = (
+        None  # bound params for the SQL (values only; no ':' prefix)
+    )
+    source: str | None = None  # System will set this to the plugin name
+    restriction_sql: str | None = (
+        None  # Optional SQL that returns (parent, child) for restriction filtering
+    )
+
+    @classmethod
+    def allow(cls, reason: str, _allow: bool = True) -> "PermissionSQL":
+        global _reason_id
+        i = _reason_id
+        _reason_id += 1
+        return cls(
+            sql=f"SELECT NULL AS parent, NULL AS child, {1 if _allow else 0} AS allow, :reason_{i} AS reason",
+            params={f"reason_{i}": reason},
+        )
+
+    @classmethod
+    def deny(cls, reason: str) -> "PermissionSQL":
+        return cls.allow(reason=reason, _allow=False)
+
+
+# This is obsolete, replaced by Action and ResourceType
+@dataclass
+class Permission:
+    name: str
+    abbr: str | None
+    description: str | None
+    takes_database: bool
+    takes_resource: bool
+    default: bool
+    # This is deliberately undocumented: it's considered an internal
+    # implementation detail for view-table/view-database and should
+    # not be used by plugins as it may change in the future.
+    implies_can_view: bool = False

datasette/plugins.py

@@ -1,24 +1,91 @@
 import importlib
+import os
 import pluggy
-import pkg_resources
+from pprint import pprint
 import sys
 from . import hookspecs
 
+if sys.version_info >= (3, 9):
+    import importlib.resources as importlib_resources
+else:
+    import importlib_resources
+if sys.version_info >= (3, 10):
+    import importlib.metadata as importlib_metadata
+else:
+    import importlib_metadata
+
+
 DEFAULT_PLUGINS = (
     "datasette.publish.heroku",
-    "datasette.publish.now",
     "datasette.publish.cloudrun",
     "datasette.facets",
+    "datasette.filters",
     "datasette.sql_functions",
+    "datasette.actor_auth_cookie",
+    "datasette.default_permissions",
+    "datasette.default_permissions.tokens",
+    "datasette.default_actions",
+    "datasette.default_column_types",
+    "datasette.default_magic_parameters",
+    "datasette.blob_renderer",
+    "datasette.default_debug_menu",
+    "datasette.default_jump_items",
+    "datasette.default_database_actions",
+    "datasette.handle_exception",
+    "datasette.forbidden",
+    "datasette.events",
 )
 
 pm = pluggy.PluginManager("datasette")
 pm.add_hookspecs(hookspecs)
 
-if not hasattr(sys, "_called_from_test"):
+DATASETTE_TRACE_PLUGINS = os.environ.get("DATASETTE_TRACE_PLUGINS", None)
+
+
+def before(hook_name, hook_impls, kwargs):
+    print(file=sys.stderr)
+    print(f"{hook_name}:", file=sys.stderr)
+    pprint(kwargs, width=40, indent=4, stream=sys.stderr)
+    print("Hook implementations:", file=sys.stderr)
+    pprint(hook_impls, width=40, indent=4, stream=sys.stderr)
+
+
+def after(outcome, hook_name, hook_impls, kwargs):
+    results = outcome.get_result()
+    if not isinstance(results, list):
+        results = [results]
+    print("Results:", file=sys.stderr)
+    pprint(results, width=40, indent=4, stream=sys.stderr)
+
+
+if DATASETTE_TRACE_PLUGINS:
+    pm.add_hookcall_monitoring(before, after)
+
+
+DATASETTE_LOAD_PLUGINS = os.environ.get("DATASETTE_LOAD_PLUGINS", None)
+
+if not hasattr(sys, "_called_from_test") and DATASETTE_LOAD_PLUGINS is None:
     # Only load plugins if not running tests
     pm.load_setuptools_entrypoints("datasette")
 
+# Load any plugins specified in DATASETTE_LOAD_PLUGINS")
+if DATASETTE_LOAD_PLUGINS is not None:
+    for package_name in [
+        name for name in DATASETTE_LOAD_PLUGINS.split(",") if name.strip()
+    ]:
+        try:
+            distribution = importlib_metadata.distribution(package_name)
+            entry_points = distribution.entry_points
+            for entry_point in entry_points:
+                if entry_point.group == "datasette":
+                    mod = entry_point.load()
+                    pm.register(mod, name=entry_point.name)
+                    # Ensure name can be found in plugin_to_distinfo later:
+                    pm._plugin_distinfo.append((mod, distribution))
+        except importlib_metadata.PackageNotFoundError:
+            sys.stderr.write("Plugin {} could not be found\n".format(package_name))
+
+
 # Load default plugins
 for plugin in DEFAULT_PLUGINS:
     mod = importlib.import_module(plugin)
@@ -31,27 +98,31 @@ def get_plugins():
     for plugin in pm.get_plugins():
         static_path = None
         templates_path = None
-        if plugin.__name__ not in DEFAULT_PLUGINS:
+        plugin_name = (
+            plugin.__name__
+            if hasattr(plugin, "__name__")
+            else plugin.__class__.__name__
+        )
+        if plugin_name not in DEFAULT_PLUGINS:
             try:
-                if pkg_resources.resource_isdir(plugin.__name__, "static"):
-                    static_path = pkg_resources.resource_filename(
-                        plugin.__name__, "static"
+                if (importlib_resources.files(plugin_name) / "static").is_dir():
+                    static_path = str(importlib_resources.files(plugin_name) / "static")
+                if (importlib_resources.files(plugin_name) / "templates").is_dir():
+                    templates_path = str(
+                        importlib_resources.files(plugin_name) / "templates"
                     )
-                if pkg_resources.resource_isdir(plugin.__name__, "templates"):
-                    templates_path = pkg_resources.resource_filename(
-                        plugin.__name__, "templates"
-                    )
-            except (KeyError, ImportError):
-                # Caused by --plugins_dir= plugins - KeyError/ImportError thrown in Py3.5
+            except (TypeError, ModuleNotFoundError):
+                # Caused by --plugins_dir= plugins
                 pass
         plugin_info = {
-            "name": plugin.__name__,
+            "name": plugin_name,
             "static_path": static_path,
             "templates_path": templates_path,
+            "hooks": [h.name for h in pm.get_hookcallers(plugin)],
         }
         distinfo = plugin_to_distinfo.get(plugin)
         if distinfo:
             plugin_info["version"] = distinfo.version
-            plugin_info["name"] = distinfo.project_name
+            plugin_info["name"] = distinfo.name or distinfo.project_name
         plugins.append(plugin_info)
     return plugins

datasette/publish/cloudrun.py

@@ -3,7 +3,7 @@ import click
 import json
 import os
 import re
-from subprocess import check_call, check_output
+from subprocess import CalledProcessError, check_call, check_output
 
 from .common import (
     add_common_publish_arguments_and_options,
@@ -23,7 +23,9 @@ def publish_subcommand(publish):
         help="Application name to use when building",
     )
     @click.option(
-        "--service", default="", help="Cloud Run service to deploy (or over-write)"
+        "--service",
+        default="",
+        help="Cloud Run service to deploy (or over-write)",
     )
     @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension")
     @click.option(
@@ -36,6 +38,51 @@ def publish_subcommand(publish):
         callback=_validate_memory,
         help="Memory to allocate in Cloud Run, e.g. 1Gi",
     )
+    @click.option(
+        "--cpu",
+        type=click.Choice(["1", "2", "4"]),
+        help="Number of vCPUs to allocate in Cloud Run",
+    )
+    @click.option(
+        "--timeout",
+        type=int,
+        help="Build timeout in seconds",
+    )
+    @click.option(
+        "--apt-get-install",
+        "apt_get_extras",
+        multiple=True,
+        help="Additional packages to apt-get install",
+    )
+    @click.option(
+        "--max-instances",
+        type=int,
+        default=1,
+        show_default=True,
+        help="Maximum Cloud Run instances (use 0 to remove the limit)",
+    )
+    @click.option(
+        "--min-instances",
+        type=int,
+        help="Minimum Cloud Run instances",
+    )
+    @click.option(
+        "--artifact-repository",
+        default="datasette",
+        show_default=True,
+        help="Artifact Registry repository to store the image",
+    )
+    @click.option(
+        "--artifact-region",
+        default="us",
+        show_default=True,
+        help="Artifact Registry location (region or multi-region)",
+    )
+    @click.option(
+        "--artifact-project",
+        default=None,
+        help="Project ID for Artifact Registry (defaults to the active project)",
+    )
     def cloudrun(
         files,
         metadata,
@@ -47,6 +94,7 @@ def publish_subcommand(publish):
         install,
         plugin_secret,
         version_note,
+        secret,
         title,
         license,
         license_url,
@@ -59,7 +107,16 @@ def publish_subcommand(publish):
         spatialite,
         show_files,
         memory,
+        cpu,
+        timeout,
+        apt_get_extras,
+        max_instances,
+        min_instances,
+        artifact_repository,
+        artifact_region,
+        artifact_project,
     ):
+        "Publish databases to Datasette running on Cloud Run"
         fail_if_publish_binary_not_installed(
             "gcloud", "Google Cloud", "https://cloud.google.com/sdk/"
         )
@@ -67,6 +124,21 @@ def publish_subcommand(publish):
             "gcloud config get-value project", shell=True, universal_newlines=True
         ).strip()
 
+        artifact_project = artifact_project or project
+
+        # Ensure Artifact Registry exists for the target image
+        _ensure_artifact_registry(
+            artifact_project=artifact_project,
+            artifact_region=artifact_region,
+            artifact_repository=artifact_repository,
+        )
+
+        artifact_host = (
+            artifact_region
+            if artifact_region.endswith("-docker.pkg.dev")
+            else f"{artifact_region}-docker.pkg.dev"
+        )
+
         if not service:
             # Show the user their current services, then prompt for one
             click.echo("Please provide a service name for this deployment\n")
@@ -84,6 +156,11 @@ def publish_subcommand(publish):
                 click.echo("")
             service = click.prompt("Service name", type=str)
 
+        image_id = (
+            f"{artifact_host}/{artifact_project}/"
+            f"{artifact_repository}/datasette-{service}"
+        )
+
         extra_metadata = {
             "title": title,
             "license": license,
@@ -94,14 +171,19 @@ def publish_subcommand(publish):
             "about_url": about_url,
         }
 
+        if not extra_options:
+            extra_options = ""
+        if "force_https_urls" not in extra_options:
+            if extra_options:
+                extra_options += " "
+            extra_options += "--setting force_https_urls on"
+
         environment_variables = {}
         if plugin_secret:
             extra_metadata["plugins"] = {}
             for plugin_name, plugin_setting, setting_value in plugin_secret:
                 environment_variable = (
-                    "{}_{}".format(plugin_name, plugin_setting)
-                    .upper()
-                    .replace("-", "_")
+                    f"{plugin_name}_{plugin_setting}".upper().replace("-", "_")
                 )
                 environment_variables[environment_variable] = setting_value
                 extra_metadata["plugins"].setdefault(plugin_name, {})[
@@ -120,27 +202,92 @@ def publish_subcommand(publish):
             install,
             spatialite,
             version_note,
+            secret,
             extra_metadata,
             environment_variables,
+            apt_get_extras=apt_get_extras,
         ):
             if show_files:
                 if os.path.exists("metadata.json"):
                     print("=== metadata.json ===\n")
-                    print(open("metadata.json").read())
+                    with open("metadata.json") as fp:
+                        print(fp.read())
                 print("\n==== Dockerfile ====\n")
-                print(open("Dockerfile").read())
+                with open("Dockerfile") as fp:
+                    print(fp.read())
                 print("\n====================\n")
 
-            image_id = "gcr.io/{project}/{name}".format(project=project, name=name)
-            check_call("gcloud builds submit --tag {}".format(image_id), shell=True)
+            check_call(
+                "gcloud builds submit --tag {}{}".format(
+                    image_id, " --timeout {}".format(timeout) if timeout else ""
+                ),
+                shell=True,
+            )
+        extra_deploy_options = []
+        for option, value in (
+            ("--memory", memory),
+            ("--cpu", cpu),
+            ("--max-instances", max_instances),
+            ("--min-instances", min_instances),
+        ):
+            if value is not None:
+                extra_deploy_options.append("{} {}".format(option, value))
         check_call(
             "gcloud run deploy --allow-unauthenticated --platform=managed --image {} {}{}".format(
-                image_id, service, " --memory {}".format(memory) if memory else ""
+                image_id,
+                service,
+                " " + " ".join(extra_deploy_options) if extra_deploy_options else "",
             ),
             shell=True,
         )
 
 
+def _ensure_artifact_registry(artifact_project, artifact_region, artifact_repository):
+    """Ensure Artifact Registry API is enabled and the repository exists."""
+
+    enable_cmd = (
+        "gcloud services enable artifactregistry.googleapis.com "
+        f"--project {artifact_project} --quiet"
+    )
+    try:
+        check_call(enable_cmd, shell=True)
+    except CalledProcessError as exc:
+        raise click.ClickException(
+            "Failed to enable artifactregistry.googleapis.com. "
+            "Please ensure you have permissions to manage services."
+        ) from exc
+
+    describe_cmd = (
+        "gcloud artifacts repositories describe {repo} --project {project} "
+        "--location {location} --quiet"
+    ).format(
+        repo=artifact_repository,
+        project=artifact_project,
+        location=artifact_region,
+    )
+    try:
+        check_call(describe_cmd, shell=True)
+        return
+    except CalledProcessError:
+        create_cmd = (
+            "gcloud artifacts repositories create {repo} --repository-format=docker "
+            '--location {location} --project {project} --description "Datasette Cloud Run images" --quiet'
+        ).format(
+            repo=artifact_repository,
+            location=artifact_region,
+            project=artifact_project,
+        )
+        try:
+            check_call(create_cmd, shell=True)
+            click.echo(f"Created Artifact Registry repository '{artifact_repository}'")
+        except CalledProcessError as exc:
+            raise click.ClickException(
+                "Failed to create Artifact Registry repository. "
+                "Use --artifact-repository/--artifact-region to point to an existing repo "
+                "or create one manually."
+            ) from exc
+
+
 def get_existing_services():
     services = json.loads(
         check_output(
@@ -156,6 +303,7 @@ def get_existing_services():
             "url": service["status"]["address"]["url"],
         }
         for service in services
+        if "url" in service["status"]
     ]
 
 

datasette/publish/common.py

@@ -1,5 +1,6 @@
 from ..utils import StaticMount
 import click
+import os
 import shutil
 import sys
 
@@ -12,13 +13,13 @@ def add_common_publish_arguments_and_options(subcommand):
                 "-m",
                 "--metadata",
                 type=click.File(mode="r"),
-                help="Path to JSON file containing metadata to publish",
+                help="Path to JSON/YAML file containing metadata to publish",
             ),
             click.option(
                 "--extra-options", help="Extra options to pass to datasette serve"
             ),
             click.option(
-                "--branch", help="Install datasette from a GitHub branch e.g. master"
+                "--branch", help="Install datasette from a GitHub branch e.g. main"
             ),
             click.option(
                 "--template-dir",
@@ -52,6 +53,12 @@ def add_common_publish_arguments_and_options(subcommand):
             click.option(
                 "--version-note", help="Additional note to show on /-/versions"
             ),
+            click.option(
+                "--secret",
+                help="Secret used for signing secure values, such as signed cookies",
+                envvar="DATASETTE_PUBLISH_SECRET",
+                default=lambda: os.urandom(32).hex(),
+            ),
             click.option("--title", help="Title for metadata"),
             click.option("--license", help="License label for metadata"),
             click.option("--license_url", help="License URL for metadata"),
@@ -78,9 +85,7 @@ def fail_if_publish_binary_not_installed(binary, publish_target, install_link):
             err=True,
         )
         click.echo(
-            "Follow the instructions at {install_link}".format(
-                install_link=install_link
-            ),
+            f"Follow the instructions at {install_link}",
             err=True,
         )
         sys.exit(1)

datasette/publish/heroku.py

@@ -3,7 +3,9 @@ from datasette import hookimpl
 import click
 import json
 import os
+import pathlib
 import shlex
+import shutil
 from subprocess import call, check_output
 import tempfile
 
@@ -11,7 +13,7 @@ from .common import (
     add_common_publish_arguments_and_options,
     fail_if_publish_binary_not_installed,
 )
-from datasette.utils import link_or_copy, link_or_copy_directory
+from datasette.utils import link_or_copy, link_or_copy_directory, parse_metadata
 
 
 @hookimpl
@@ -24,6 +26,15 @@ def publish_subcommand(publish):
         default="datasette",
         help="Application name to use when deploying",
     )
+    @click.option(
+        "--tar",
+        help="--tar option to pass to Heroku, e.g. --tar=/usr/local/bin/gtar",
+    )
+    @click.option(
+        "--generate-dir",
+        type=click.Path(dir_okay=True, file_okay=False),
+        help="Output generated application files and stop without deploying",
+    )
     def heroku(
         files,
         metadata,
@@ -35,6 +46,7 @@ def publish_subcommand(publish):
         install,
         plugin_secret,
         version_note,
+        secret,
         title,
         license,
         license_url,
@@ -43,7 +55,10 @@ def publish_subcommand(publish):
         about,
         about_url,
         name,
+        tar,
+        generate_dir,
     ):
+        "Publish databases to Datasette running on Heroku"
         fail_if_publish_binary_not_installed(
             "heroku", "Heroku", "https://cli.heroku.com"
         )
@@ -72,17 +87,12 @@ def publish_subcommand(publish):
             "about_url": about_url,
         }
 
-        environment_variables = {
-            # Avoid uvicorn error: https://github.com/simonw/datasette/issues/633
-            "WEB_CONCURRENCY": "1"
-        }
+        environment_variables = {}
         if plugin_secret:
             extra_metadata["plugins"] = {}
             for plugin_name, plugin_setting, setting_value in plugin_secret:
                 environment_variable = (
-                    "{}_{}".format(plugin_name, plugin_setting)
-                    .upper()
-                    .replace("-", "_")
+                    f"{plugin_name}_{plugin_setting}".upper().replace("-", "_")
                 )
                 environment_variables[environment_variable] = setting_value
                 extra_metadata["plugins"].setdefault(plugin_name, {})[
@@ -100,8 +110,19 @@ def publish_subcommand(publish):
             static,
             install,
             version_note,
+            secret,
             extra_metadata,
         ):
+            if generate_dir:
+                # Recursively copy files from current working directory to it
+                if pathlib.Path(generate_dir).exists():
+                    raise click.ClickException("Directory already exists")
+                shutil.copytree(".", generate_dir)
+                click.echo(
+                    f"Generated files written to {generate_dir}, stopping without deploying",
+                    err=True,
+                )
+                return
             app_name = None
             if name:
                 # Check to see if this app already exists
@@ -125,11 +146,14 @@ def publish_subcommand(publish):
                 app_name = json.loads(create_output)["name"]
 
             for key, value in environment_variables.items():
-                call(
-                    ["heroku", "config:set", "-a", app_name, "{}={}".format(key, value)]
-                )
-
-            call(["heroku", "builds:create", "-a", app_name, "--include-vcs-ignore"])
+                call(["heroku", "config:set", "-a", app_name, f"{key}={value}"])
+            tar_option = []
+            if tar:
+                tar_option = ["--tar", tar]
+            call(
+                ["heroku", "builds:create", "-a", app_name, "--include-vcs-ignore"]
+                + tar_option
+            )
 
 
 @contextmanager
@@ -144,6 +168,7 @@ def temporary_heroku_directory(
     static,
     install,
     version_note,
+    secret,
     extra_metadata=None,
 ):
     extra_metadata = extra_metadata or {}
@@ -154,7 +179,7 @@ def temporary_heroku_directory(
     file_names = [os.path.split(f)[-1] for f in files]
 
     if metadata:
-        metadata_content = json.load(metadata)
+        metadata_content = parse_metadata(metadata.read())
     else:
         metadata_content = {}
     for key, value in extra_metadata.items():
@@ -165,24 +190,24 @@ def temporary_heroku_directory(
         os.chdir(tmp.name)
 
         if metadata_content:
-            open("metadata.json", "w").write(json.dumps(metadata_content, indent=2))
+            with open("metadata.json", "w") as fp:
+                fp.write(json.dumps(metadata_content, indent=2))
 
-        open("runtime.txt", "w").write("python-3.8.0")
+        with open("runtime.txt", "w") as fp:
+            fp.write("python-3.11.0")
 
         if branch:
             install = [
-                "https://github.com/simonw/datasette/archive/{branch}.zip".format(
-                    branch=branch
-                )
+                f"https://github.com/simonw/datasette/archive/{branch}.zip"
             ] + list(install)
         else:
             install = ["datasette"] + list(install)
 
-        open("requirements.txt", "w").write("\n".join(install))
+        with open("requirements.txt", "w") as fp:
+            fp.write("\n".join(install))
         os.mkdir("bin")
-        open("bin/post_compile", "w").write(
-            "datasette inspect --inspect-file inspect-data.json"
-        )
+        with open("bin/post_compile", "w") as fp:
+            fp.write("datasette inspect --inspect-file inspect-data.json")
 
         extras = []
         if template_dir:
@@ -206,7 +231,7 @@ def temporary_heroku_directory(
             link_or_copy_directory(
                 os.path.join(saved_cwd, path), os.path.join(tmp.name, mount_point)
             )
-            extras.extend(["--static", "{}:{}".format(mount_point, mount_point)])
+            extras.extend(["--static", f"{mount_point}:{mount_point}"])
 
         quoted_files = " ".join(
             ["-i {}".format(shlex.quote(file_name)) for file_name in file_names]
@@ -214,7 +239,8 @@ def temporary_heroku_directory(
         procfile_cmd = "web: datasette serve --host 0.0.0.0 {quoted_files} --cors --port $PORT --inspect-file inspect-data.json {extras}".format(
             quoted_files=quoted_files, extras=" ".join(extras)
         )
-        open("Procfile", "w").write(procfile_cmd)
+        with open("Procfile", "w") as fp:
+            fp.write(procfile_cmd)
 
         for path, filename in zip(file_paths, file_names):
             link_or_copy(path, os.path.join(tmp.name, filename))

datasette/publish/now.py

@@ -1,133 +0,0 @@
-from datasette import hookimpl
-import click
-import json
-import os
-from subprocess import run, PIPE
-
-from .common import (
-    add_common_publish_arguments_and_options,
-    fail_if_publish_binary_not_installed,
-)
-from ..utils import temporary_docker_directory
-
-
-@hookimpl
-def publish_subcommand(publish):
-    @publish.command()
-    @add_common_publish_arguments_and_options
-    @click.option(
-        "-n",
-        "--name",
-        default="datasette",
-        help="Application name to use when deploying",
-    )
-    @click.option("--force", is_flag=True, help="Pass --force option to now")
-    @click.option("--token", help="Auth token to use for deploy")
-    @click.option("--alias", multiple=True, help="Desired alias e.g. yoursite.now.sh")
-    @click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension")
-    @click.option(
-        "--show-files",
-        is_flag=True,
-        help="Output the generated Dockerfile and metadata.json",
-    )
-    def nowv1(
-        files,
-        metadata,
-        extra_options,
-        branch,
-        template_dir,
-        plugins_dir,
-        static,
-        install,
-        plugin_secret,
-        version_note,
-        title,
-        license,
-        license_url,
-        source,
-        source_url,
-        about,
-        about_url,
-        name,
-        force,
-        token,
-        alias,
-        spatialite,
-        show_files,
-    ):
-        fail_if_publish_binary_not_installed("now", "Zeit Now", "https://zeit.co/now")
-        if extra_options:
-            extra_options += " "
-        else:
-            extra_options = ""
-        extra_options += "--config force_https_urls:on"
-
-        extra_metadata = {
-            "title": title,
-            "license": license,
-            "license_url": license_url,
-            "source": source,
-            "source_url": source_url,
-            "about": about,
-            "about_url": about_url,
-        }
-
-        environment_variables = {}
-        if plugin_secret:
-            extra_metadata["plugins"] = {}
-            for plugin_name, plugin_setting, setting_value in plugin_secret:
-                environment_variable = (
-                    "{}_{}".format(plugin_name, plugin_setting)
-                    .upper()
-                    .replace("-", "_")
-                )
-                environment_variables[environment_variable] = setting_value
-                extra_metadata["plugins"].setdefault(plugin_name, {})[
-                    plugin_setting
-                ] = {"$env": environment_variable}
-
-        with temporary_docker_directory(
-            files,
-            name,
-            metadata,
-            extra_options,
-            branch,
-            template_dir,
-            plugins_dir,
-            static,
-            install,
-            spatialite,
-            version_note,
-            extra_metadata,
-            environment_variables,
-        ):
-            now_json = {"version": 1}
-            open("now.json", "w").write(json.dumps(now_json, indent=4))
-            args = []
-            if force:
-                args.append("--force")
-            if token:
-                args.append("--token={}".format(token))
-            if args:
-                done = run(["now"] + args, stdout=PIPE)
-            else:
-                done = run("now", stdout=PIPE)
-            deployment_url = done.stdout
-            if show_files:
-                if os.path.exists("metadata.json"):
-                    print("=== metadata.json ===\n")
-                    print(open("metadata.json").read())
-                print("\n==== Dockerfile ====\n")
-                print(open("Dockerfile").read())
-                print("\n====================\n")
-            if alias:
-                # I couldn't get --target=production working, so I call
-                # 'now alias' with arguments directly instead - but that
-                # means I need to figure out what URL it was deployed to.
-                for single_alias in alias:  # --alias can be specified multiple times
-                    args = ["now", "alias", deployment_url, single_alias]
-                    if token:
-                        args.append("--token={}".format(token))
-                    run(args)
-            else:
-                print(deployment_url.decode("latin1"))

datasette/renderer.py

@@ -4,7 +4,9 @@ from datasette.utils import (
     remove_infinites,
     CustomJSONEncoder,
     path_from_row_pks,
+    sqlite3,
 )
+from datasette.utils.asgi import Response
 
 
 def convert_specific_columns_to_json(rows, columns, json_cols):
@@ -18,21 +20,21 @@ def convert_specific_columns_to_json(rows, columns, json_cols):
             if column in json_cols:
                 try:
                     value = json.loads(value)
-                except (TypeError, ValueError) as e:
-                    print(e)
+                except (TypeError, ValueError):
                     pass
             new_row.append(value)
         new_rows.append(new_row)
     return new_rows
 
 
-def json_renderer(args, data, view_name):
-    """ Render a response as JSON """
+def json_renderer(request, args, data, error, truncated=None):
+    """Render a response as JSON"""
     status_code = 200
+
     # Handle the _json= parameter which may modify data["rows"]
     json_cols = []
     if "_json" in args:
-        json_cols = args["_json"]
+        json_cols = args.getlist("_json")
     if json_cols and "rows" in data and "columns" in data:
         data["rows"] = convert_specific_columns_to_json(
             data["rows"], data["columns"], json_cols
@@ -43,22 +45,38 @@ def json_renderer(args, data, view_name):
         data["rows"] = [remove_infinites(row) for row in data["rows"]]
 
     # Deal with the _shape option
-    shape = args.get("_shape", "arrays")
+    shape = args.get("_shape", "objects")
+    # if there's an error, ignore the shape entirely
+    data["ok"] = True
+    if error:
+        shape = "objects"
+        status_code = 400
+        data["error"] = error
+        data["ok"] = False
+
+    if truncated is not None:
+        data["truncated"] = truncated
     if shape == "arrayfirst":
-        data = [row[0] for row in data["rows"]]
+        if not data["rows"]:
+            data = []
+        elif isinstance(data["rows"][0], sqlite3.Row):
+            data = [row[0] for row in data["rows"]]
+        else:
+            assert isinstance(data["rows"][0], dict)
+            data = [next(iter(row.values())) for row in data["rows"]]
     elif shape in ("objects", "object", "array"):
         columns = data.get("columns")
         rows = data.get("rows")
-        if rows and columns:
+        if rows and columns and not isinstance(rows[0], dict):
             data["rows"] = [dict(zip(columns, row)) for row in rows]
         if shape == "object":
-            error = None
+            shape_error = None
             if "primary_keys" not in data:
-                error = "_shape=object is only available on tables"
+                shape_error = "_shape=object is only available on tables"
             else:
                 pks = data["primary_keys"]
                 if not pks:
-                    error = (
+                    shape_error = (
                         "_shape=object not available for tables with no primary keys"
                     )
                 else:
@@ -67,26 +85,41 @@ def json_renderer(args, data, view_name):
                         pk_string = path_from_row_pks(row, pks, not pks)
                         object_rows[pk_string] = row
                     data = object_rows
-            if error:
-                data = {"ok": False, "error": error}
+            if shape_error:
+                data = {"ok": False, "error": shape_error}
         elif shape == "array":
             data = data["rows"]
+
     elif shape == "arrays":
-        pass
+        if not data["rows"]:
+            pass
+        elif isinstance(data["rows"][0], sqlite3.Row):
+            data["rows"] = [list(row) for row in data["rows"]]
+        else:
+            data["rows"] = [list(row.values()) for row in data["rows"]]
     else:
         status_code = 400
         data = {
             "ok": False,
-            "error": "Invalid _shape: {}".format(shape),
+            "error": f"Invalid _shape: {shape}",
             "status": 400,
             "title": None,
         }
+
+    # Don't include "columns" in output
+    # https://github.com/simonw/datasette/issues/2136
+    if isinstance(data, dict) and "columns" not in request.args.getlist("_extra"):
+        data.pop("columns", None)
+
     # Handle _nl option for _shape=array
     nl = args.get("_nl", "")
     if nl and shape == "array":
-        body = "\n".join(json.dumps(item) for item in data)
+        body = "\n".join(json.dumps(item, cls=CustomJSONEncoder) for item in data)
         content_type = "text/plain"
     else:
         body = json.dumps(data, cls=CustomJSONEncoder)
         content_type = "application/json; charset=utf-8"
-    return {"body": body, "status_code": status_code, "content_type": content_type}
+    headers = {}
+    return Response(
+        body, status=status_code, headers=headers, content_type=content_type
+    )

datasette/resources.py

@@ -0,0 +1,58 @@
+"""Core resource types for Datasette's permission system."""
+
+from datasette.permissions import Resource
+
+
+class DatabaseResource(Resource):
+    """A database in Datasette."""
+
+    name = "database"
+    parent_class = None  # Top of the resource hierarchy
+
+    def __init__(self, database: str):
+        super().__init__(parent=database, child=None)
+
+    @classmethod
+    async def resources_sql(cls, datasette, actor=None) -> str:
+        return """
+            SELECT database_name AS parent, NULL AS child
+            FROM catalog_databases
+        """
+
+
+class TableResource(Resource):
+    """A table in a database."""
+
+    name = "table"
+    parent_class = DatabaseResource
+
+    def __init__(self, database: str, table: str):
+        super().__init__(parent=database, child=table)
+
+    @classmethod
+    async def resources_sql(cls, datasette, actor=None) -> str:
+        return """
+            SELECT database_name AS parent, table_name AS child
+            FROM catalog_tables
+            UNION ALL
+            SELECT database_name AS parent, view_name AS child
+            FROM catalog_views
+        """
+
+
+class QueryResource(Resource):
+    """A stored query in a database."""
+
+    name = "query"
+    parent_class = DatabaseResource
+
+    def __init__(self, database: str, query: str):
+        super().__init__(parent=database, child=query)
+
+    @classmethod
+    async def resources_sql(cls, datasette, actor=None) -> str:
+        return """
+            SELECT q.database_name AS parent, q.name AS child
+            FROM queries q
+            JOIN catalog_databases cd ON cd.database_name = q.database_name
+        """

datasette/static/cm-editor-6.0.1.js

@@ -0,0 +1,74 @@
+import { EditorView, basicSetup } from "codemirror";
+import { keymap } from "@codemirror/view";
+import { sql, SQLDialect } from "@codemirror/lang-sql";
+
+// A variation of SQLite from lang-sql https://github.com/codemirror/lang-sql/blob/ebf115fffdbe07f91465ccbd82868c587f8182bc/src/sql.ts#L231
+const SQLite = SQLDialect.define({
+  // Based on https://www.sqlite.org/lang_keywords.html based on likely keywords to be used in select queries
+  // https://github.com/simonw/datasette/pull/1893#issuecomment-1316401895:
+  keywords:
+    "and as asc between by case cast count current_date current_time current_timestamp desc distinct each else escape except exists explain filter first for from full generated group having if in index inner intersect into isnull join last left like limit not null or order outer over pragma primary query raise range regexp right rollback row select set table then to union unique using values view virtual when where",
+  // https://www.sqlite.org/datatype3.html
+  types: "null integer real text blob",
+  builtin: "",
+  operatorChars: "*+-%<>!=&|/~",
+  identifierQuotes: '`"',
+  specialVar: "@:?$",
+});
+
+// Utility function from https://codemirror.net/docs/migration/
+export function editorFromTextArea(textarea, conf = {}) {
+  // This could also be configured with a set of tables and columns for better autocomplete:
+  // https://github.com/codemirror/lang-sql#user-content-sqlconfig.tables
+  let view = new EditorView({
+    doc: textarea.value,
+    extensions: [
+      keymap.of([
+        {
+          key: "Shift-Enter",
+          run: function () {
+            textarea.value = view.state.doc.toString();
+            textarea.form.submit();
+            return true;
+          },
+        },
+        {
+          key: "Meta-Enter",
+          run: function () {
+            textarea.value = view.state.doc.toString();
+            textarea.form.submit();
+            return true;
+          },
+        },
+      ]),
+      // This has to be after the keymap or else the basicSetup keys will prevent
+      // Meta-Enter from running
+      basicSetup,
+      EditorView.lineWrapping,
+      sql({
+        dialect: SQLite,
+        schema: conf.schema,
+        tables: conf.tables,
+        defaultTableName: conf.defaultTableName,
+        defaultSchemaName: conf.defaultSchemaName,
+      }),
+    ],
+  });
+
+  // Idea taken from https://discuss.codemirror.net/t/resizing-codemirror-6/3265.
+  // Using CSS resize: both and scheduling a measurement when the element changes.
+  let editorDOM = view.contentDOM.closest(".cm-editor");
+  let observer = new ResizeObserver(function () {
+    view.requestMeasure();
+  });
+  observer.observe(editorDOM, { attributes: true });
+
+  textarea.parentNode.insertBefore(view.dom, textarea);
+  textarea.style.display = "none";
+  if (textarea.form) {
+    textarea.form.addEventListener("submit", () => {
+      textarea.value = view.state.doc.toString();
+    });
+  }
+  return view;
+}

datasette/static/column-chooser.js

@@ -0,0 +1,699 @@
+class ColumnChooser extends HTMLElement {
+  constructor() {
+    super();
+    this.attachShadow({ mode: "open" });
+
+    // State
+    this._items = [];
+    this._checked = new Set();
+    this._savedItems = null;
+    this._savedChecked = null;
+    this._onApply = null;
+
+    // Drag state
+    this._ghost = null;
+    this._dragSrcIdx = null;
+    this._dropTargetIdx = null;
+    this._dropPosition = null;
+    this._ghostOffX = 0;
+    this._ghostOffY = 0;
+    this._autoScrollRAF = null;
+    this._lastPointerY = 0;
+    this._lastPointerX = 0;
+    this._SCROLL_ZONE = 72;
+    this._SCROLL_SPEED = 0.4;
+
+    // Bound handlers
+    this._onMove = this._onMove.bind(this);
+    this._onUp = this._onUp.bind(this);
+
+    this.shadowRoot.innerHTML = `
+      <style>
+        :host {
+          --ink: #0f0f0f;
+          --paper: #f5f3ef;
+          --muted: #6b6b6b;
+          --rule: #e2dfd8;
+          --accent: #1a56db;
+          --accent-light: #e8effd;
+          --card: #ffffff;
+        }
+
+        * { box-sizing: border-box; margin: 0; padding: 0; }
+
+        dialog {
+          border: none;
+          border-radius: var(--modal-border-radius, 0.75rem);
+          padding: 0;
+          margin: auto;
+          width: 100%;
+          max-width: 420px;
+          max-height: min(640px, calc(100vh - 32px));
+          box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
+          animation: slideIn var(--modal-animation-duration, 0.2s) ease-out;
+          overflow: hidden;
+          font-family: system-ui, -apple-system, sans-serif;
+          background: var(--card);
+          -webkit-user-select: none;
+          -webkit-touch-callout: none;
+          -webkit-tap-highlight-color: transparent;
+        }
+
+        dialog[open] {
+          display: flex;
+          flex-direction: column;
+          height: min(640px, calc(100vh - 32px));
+        }
+
+        dialog::backdrop {
+          background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
+          backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+          -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+          animation: fadeIn var(--modal-animation-duration, 0.2s) ease-out;
+        }
+
+        @keyframes slideIn {
+          from {
+            opacity: 0;
+            transform: translateY(-20px) scale(0.95);
+          }
+          to {
+            opacity: 1;
+            transform: translateY(0) scale(1);
+          }
+        }
+
+        @keyframes fadeIn {
+          from { opacity: 0; }
+          to { opacity: 1; }
+        }
+
+        .modal-header {
+          padding: 20px 24px 16px;
+          border-bottom: 1px solid var(--rule);
+          display: flex;
+          align-items: center;
+          justify-content: space-between;
+          flex-shrink: 0;
+        }
+
+        .modal-title {
+          font-size: 1rem;
+          font-weight: 600;
+        }
+
+        .modal-meta {
+          font-family: ui-monospace, monospace;
+          font-size: 0.7rem;
+          color: var(--muted);
+          background: var(--paper);
+          padding: 3px 9px;
+          border-radius: 20px;
+        }
+
+        .list-toolbar {
+          padding: 6px 24px;
+          border-bottom: 1px solid var(--rule);
+          display: flex;
+          gap: 12px;
+          flex-shrink: 0;
+        }
+
+        .list-toolbar button {
+          background: var(--accent-light);
+          border: 1px solid var(--rule);
+          border-radius: 4px;
+          font-family: inherit;
+          font-size: 0.75rem;
+          color: var(--accent);
+          cursor: pointer;
+          padding: 3px 10px;
+          transition: background 0.12s, color 0.12s;
+        }
+        .list-toolbar button:hover { background: var(--accent); color: white; }
+
+        .list-wrap {
+          flex: 1;
+          overflow-y: auto;
+          overflow-x: hidden;
+          position: relative;
+          overscroll-behavior: contain;
+          -webkit-overflow-scrolling: touch;
+        }
+
+        .list-wrap::before,
+        .list-wrap::after {
+          content: '';
+          position: sticky;
+          display: block;
+          left: 0; right: 0;
+          height: 20px;
+          pointer-events: none;
+          z-index: 5;
+          transition: opacity 0.2s;
+        }
+        .list-wrap::before {
+          top: 0;
+          background: linear-gradient(to bottom, rgba(255,255,255,0.9), transparent);
+        }
+        .list-wrap::after {
+          bottom: 0;
+          background: linear-gradient(to top, rgba(255,255,255,0.9), transparent);
+          margin-top: -20px;
+        }
+
+        .scroll-zone {
+          position: absolute;
+          left: 0; right: 0;
+          height: 72px;
+          pointer-events: none;
+          z-index: 10;
+        }
+        .scroll-zone-top { top: 0; }
+        .scroll-zone-bot { bottom: 0; }
+
+        .drag-list {
+          list-style: none;
+          padding: 4px 0;
+        }
+
+        .drag-item {
+          display: flex;
+          align-items: center;
+          background: white;
+          border-bottom: 1px solid var(--rule);
+          user-select: none;
+          -webkit-user-select: none;
+          -webkit-touch-callout: none;
+          position: relative;
+          transition: background 0.08s;
+        }
+
+        .drag-item:last-child { border-bottom: none; }
+
+        .drag-handle {
+          display: flex;
+          align-items: center;
+          justify-content: center;
+          width: 48px;
+          height: 48px;
+          flex-shrink: 0;
+          cursor: grab;
+          color: #c8c4bc;
+          touch-action: none;
+          transition: color 0.15s;
+        }
+
+        .drag-handle:hover { color: var(--accent); }
+        .drag-handle svg { pointer-events: none; display: block; }
+
+        .drag-item-content {
+          display: flex;
+          align-items: center;
+          flex: 1;
+          min-width: 0;
+          cursor: pointer;
+        }
+
+        .drag-item-check {
+          display: flex;
+          align-items: center;
+          width: 32px;
+          height: 48px;
+          flex-shrink: 0;
+        }
+
+        .drag-item-check input[type="checkbox"] {
+          width: 16px;
+          height: 16px;
+          accent-color: var(--accent);
+          cursor: pointer;
+        }
+
+        .drag-item-label {
+          flex: 1;
+          font-size: 0.9rem;
+          line-height: 48px;
+          padding-right: 16px;
+          white-space: nowrap;
+          overflow: hidden;
+          text-overflow: ellipsis;
+          cursor: default;
+        }
+
+        .drag-item.is-dragging {
+          opacity: 0;
+        }
+
+        .drop-indicator {
+          position: absolute;
+          left: 48px;
+          right: 0;
+          height: 2px;
+          background: var(--accent);
+          border-radius: 99px;
+          pointer-events: none;
+          z-index: 20;
+          display: none;
+        }
+        .drop-indicator.top { top: -1px; display: block; }
+        .drop-indicator.bottom { bottom: -1px; display: block; }
+
+        .drag-ghost {
+          position: fixed;
+          pointer-events: none;
+          z-index: 9999;
+          background: white;
+          border-radius: 6px;
+          box-shadow: 0 8px 32px rgba(0,0,0,0.18), 0 2px 8px rgba(0,0,0,0.1);
+          display: flex;
+          align-items: center;
+          border: 1.5px solid var(--accent-light);
+          opacity: 0.97;
+          will-change: transform;
+          font-family: system-ui, -apple-system, sans-serif;
+        }
+
+        .scroll-pulse {
+          position: absolute;
+          left: 50%;
+          transform: translateX(-50%);
+          width: 32px;
+          height: 32px;
+          border-radius: 50%;
+          background: var(--accent);
+          opacity: 0;
+          pointer-events: none;
+          z-index: 10;
+          transition: opacity 0.15s;
+        }
+        .scroll-pulse.top { top: 8px; }
+        .scroll-pulse.bot { bottom: 8px; }
+        .scroll-pulse.active {
+          opacity: 0.18;
+          animation: pulse 0.8s ease-in-out infinite;
+        }
+
+        @keyframes pulse {
+          0%, 100% { transform: translateX(-50%) scale(1); opacity: 0.18; }
+          50% { transform: translateX(-50%) scale(1.5); opacity: 0.07; }
+        }
+
+        .modal-footer {
+          padding: 14px 20px;
+          border-top: 1px solid var(--rule);
+          display: flex;
+          align-items: center;
+          gap: 10px;
+          flex-shrink: 0;
+          background: var(--paper);
+        }
+
+        .footer-info {
+          flex: 1;
+          font-family: ui-monospace, monospace;
+          font-size: 0.68rem;
+          color: var(--muted);
+        }
+
+        .btn {
+          border: none;
+          border-radius: 5px;
+          padding: 9px 20px;
+          font-size: 0.85rem;
+          font-weight: 500;
+          cursor: pointer;
+          touch-action: manipulation;
+          font-family: inherit;
+          transition: background 0.12s;
+        }
+
+        .btn-primary {
+          background: var(--accent);
+          color: white;
+        }
+        .btn-primary:hover { background: #1448c0; }
+
+        .btn-ghost {
+          background: transparent;
+          color: var(--muted);
+          border: 1px solid var(--rule);
+        }
+        .btn-ghost:hover { background: var(--rule); color: var(--ink); }
+
+        .list-wrap::-webkit-scrollbar { width: 5px; }
+        .list-wrap::-webkit-scrollbar-track { background: transparent; }
+        .list-wrap::-webkit-scrollbar-thumb { background: var(--rule); border-radius: 99px; }
+
+        input, textarea { -webkit-user-select: auto; user-select: auto; }
+      </style>
+
+      <dialog aria-labelledby="modalTitle">
+          <div class="modal-header">
+            <span class="modal-title" id="modalTitle">Choose columns</span>
+            <span class="modal-meta" id="selectedCount"></span>
+          </div>
+          <div class="list-toolbar">
+            <button id="selectAllBtn">Select all</button>
+            <button id="deselectAllBtn">Deselect all</button>
+          </div>
+          <div class="list-wrap" id="listWrap">
+            <div class="scroll-pulse top" id="pulseTop"></div>
+            <div class="scroll-pulse bot" id="pulseBot"></div>
+            <ul class="drag-list" id="dragList"></ul>
+          </div>
+          <div class="modal-footer">
+            <span class="footer-info" id="footerInfo"></span>
+            <button class="btn btn-ghost" id="cancelBtn">Cancel</button>
+            <button class="btn btn-primary" id="applyBtn">Apply</button>
+          </div>
+      </dialog>
+    `;
+
+    // DOM refs
+    this._dialog = this.shadowRoot.querySelector("dialog");
+    this._listWrap = this.shadowRoot.getElementById("listWrap");
+    this._dragList = this.shadowRoot.getElementById("dragList");
+    this._pulseTop = this.shadowRoot.getElementById("pulseTop");
+    this._pulseBot = this.shadowRoot.getElementById("pulseBot");
+    this._selectAllBtn = this.shadowRoot.getElementById("selectAllBtn");
+    this._deselectAllBtn = this.shadowRoot.getElementById("deselectAllBtn");
+    this._cancelBtn = this.shadowRoot.getElementById("cancelBtn");
+    this._applyBtn = this.shadowRoot.getElementById("applyBtn");
+    this._countEl = this.shadowRoot.getElementById("selectedCount");
+    this._footerEl = this.shadowRoot.getElementById("footerInfo");
+
+    // Event listeners
+    this._selectAllBtn.addEventListener("click", () => this._selectAll());
+    this._deselectAllBtn.addEventListener("click", () => this._deselectAll());
+    this._cancelBtn.addEventListener("click", () => this._close());
+    this._applyBtn.addEventListener("click", () => this._apply());
+    this._dialog.addEventListener("click", (e) => {
+      if (e.target === this._dialog) this._close();
+    });
+    this._dialog.addEventListener("cancel", (e) => {
+      e.preventDefault();
+      this._close();
+    });
+  }
+
+  /**
+   * Open the column chooser dialog.
+   * @param {Object} opts
+   * @param {string[]} opts.columns - All available column names, in display order.
+   * @param {string[]} opts.selected - Column names that should be pre-checked.
+   * @param {function(string[]): void} opts.onApply - Called with the selected columns in order when Apply is clicked.
+   */
+  open({ columns, selected = [], onApply }) {
+    this._items = [...columns];
+    this._checked = new Set(selected);
+    this._onApply = onApply || null;
+
+    // Save state for cancel/restore
+    this._savedItems = [...this._items];
+    this._savedChecked = new Set(this._checked);
+
+    this._render();
+    this._dialog.showModal();
+  }
+
+  // ── Internal methods ──
+
+  _close() {
+    this._items = this._savedItems ? [...this._savedItems] : this._items;
+    this._checked = this._savedChecked
+      ? new Set(this._savedChecked)
+      : this._checked;
+    this._dialog.close();
+  }
+
+  _selectAll() {
+    this._items.forEach((col) => this._checked.add(col));
+    this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => {
+      cb.checked = true;
+    });
+    this._updateCounts();
+  }
+
+  _deselectAll() {
+    this._checked.clear();
+    this._dragList.querySelectorAll('input[type="checkbox"]').forEach((cb) => {
+      cb.checked = false;
+    });
+    this._updateCounts();
+  }
+
+  _apply() {
+    const selected = this._items.filter((col) => this._checked.has(col));
+    this._dialog.close();
+    if (this._onApply) {
+      this._onApply(selected);
+    }
+  }
+
+  _render() {
+    this._dragList.innerHTML = "";
+    this._items.forEach((col, i) => {
+      const li = document.createElement("li");
+      li.className = "drag-item";
+      li.dataset.idx = i;
+      li.innerHTML = `
+        <span class="drag-handle" aria-label="Drag to reorder">
+          <svg width="12" height="18" viewBox="0 0 12 18" fill="currentColor">
+            <circle cx="3.5" cy="3.5" r="1.8"/>
+            <circle cx="8.5" cy="3.5" r="1.8"/>
+            <circle cx="3.5" cy="9" r="1.8"/>
+            <circle cx="8.5" cy="9" r="1.8"/>
+            <circle cx="3.5" cy="14.5" r="1.8"/>
+            <circle cx="8.5" cy="14.5" r="1.8"/>
+          </svg>
+        </span>
+        <label class="drag-item-content">
+          <span class="drag-item-check">
+            <input type="checkbox" ${this._checked.has(col) ? "checked" : ""}>
+          </span>
+          <span class="drag-item-label">${col}</span>
+        </label>
+        <div class="drop-indicator"></div>
+      `;
+
+      li.querySelector("input").addEventListener("change", (e) => {
+        e.target.checked ? this._checked.add(col) : this._checked.delete(col);
+        this._updateCounts();
+      });
+
+      li.querySelector(".drag-handle").addEventListener("pointerdown", (e) =>
+        this._startDrag(e, i),
+      );
+      this._dragList.appendChild(li);
+    });
+
+    this._updateCounts();
+  }
+
+  _updateCounts() {
+    const n = this._checked.size;
+    this._countEl.textContent = `${n} of ${this._items.length} selected`;
+    this._footerEl.textContent = `${this._items.length} columns`;
+  }
+
+  // ── Drag engine ──
+
+  _startDrag(e, idx) {
+    e.preventDefault();
+    this._dragSrcIdx = idx;
+
+    const srcEl = this._dragList.children[idx];
+    const rect = srcEl.getBoundingClientRect();
+
+    this._ghostOffX = e.clientX - rect.left;
+    this._ghostOffY = e.clientY - rect.top;
+
+    // Build ghost inside shadow DOM
+    this._ghost = document.createElement("div");
+    this._ghost.className = "drag-ghost";
+    this._ghost.style.width = rect.width + "px";
+    this._ghost.style.height = rect.height + "px";
+    this._ghost.innerHTML = srcEl.innerHTML;
+    this._ghost.querySelector(".drop-indicator")?.remove();
+    const h = this._ghost.querySelector(".drag-handle");
+    if (h) h.style.color = "var(--accent)";
+    this.shadowRoot.appendChild(this._ghost);
+
+    srcEl.classList.add("is-dragging");
+    this._positionGhost(e.clientX, e.clientY);
+
+    document.addEventListener("pointermove", this._onMove);
+    document.addEventListener("pointerup", this._onUp);
+    document.addEventListener("pointercancel", this._onUp);
+  }
+
+  _positionGhost(cx, cy) {
+    this._ghost.style.left = cx - this._ghostOffX + "px";
+    this._ghost.style.top = cy - this._ghostOffY + "px";
+  }
+
+  _onMove(e) {
+    this._lastPointerX = e.clientX;
+    this._lastPointerY = e.clientY;
+    this._positionGhost(e.clientX, e.clientY);
+    this._updateDropTarget(e.clientY);
+    this._updateAutoScroll(e.clientY);
+  }
+
+  _onUp() {
+    document.removeEventListener("pointermove", this._onMove);
+    document.removeEventListener("pointerup", this._onUp);
+    document.removeEventListener("pointercancel", this._onUp);
+
+    this._stopAutoScroll();
+
+    const noMove =
+      this._dropTargetIdx === null || this._dropTargetIdx === this._dragSrcIdx;
+    this._clearDropIndicators();
+
+    let dest = null;
+    if (!noMove) {
+      const moved = this._items.splice(this._dragSrcIdx, 1)[0];
+      dest = this._dropTargetIdx;
+      if (this._dropPosition === "after") dest++;
+      if (dest > this._dragSrcIdx) dest--;
+      this._items.splice(dest, 0, moved);
+    }
+
+    this._dragSrcIdx = null;
+    this._dropTargetIdx = null;
+    this._dropPosition = null;
+
+    const g = this._ghost;
+    this._ghost = null;
+
+    if (noMove) {
+      if (g) g.remove();
+      this._render();
+      return;
+    }
+
+    this._render();
+
+    if (g && dest !== null) {
+      const landedEl = this._dragList.children[dest];
+      if (landedEl) {
+        landedEl.style.opacity = "0";
+        const r = landedEl.getBoundingClientRect();
+        g.getBoundingClientRect();
+        g.style.transition =
+          "left 0.15s cubic-bezier(0.22, 1, 0.36, 1), top 0.15s cubic-bezier(0.22, 1, 0.36, 1), box-shadow 0.15s, opacity 0.1s 0.1s";
+        g.style.left = r.left + "px";
+        g.style.top = r.top + "px";
+        g.style.boxShadow = "0 1px 4px rgba(0,0,0,0.08)";
+        g.style.opacity = "0";
+        setTimeout(() => {
+          g.remove();
+          if (landedEl) landedEl.style.opacity = "";
+        }, 160);
+      } else {
+        g.remove();
+      }
+    } else if (g) {
+      g.remove();
+    }
+  }
+
+  _updateDropTarget(clientY) {
+    this._clearDropIndicators();
+    const listItems = [
+      ...this._dragList.querySelectorAll(".drag-item:not(.is-dragging)"),
+    ];
+    if (!listItems.length) return;
+
+    let best = null,
+      bestDist = Infinity;
+    listItems.forEach((li) => {
+      const r = li.getBoundingClientRect();
+      const mid = r.top + r.height / 2;
+      const dist = Math.abs(clientY - mid);
+      if (dist < bestDist) {
+        bestDist = dist;
+        best = li;
+      }
+    });
+
+    if (!best) return;
+    const r = best.getBoundingClientRect();
+    const mid = r.top + r.height / 2;
+    const above = clientY < mid;
+    const indic = best.querySelector(".drop-indicator");
+
+    this._dropTargetIdx = parseInt(best.dataset.idx);
+    this._dropPosition = above ? "before" : "after";
+
+    if (indic) {
+      indic.className = "drop-indicator " + (above ? "top" : "bottom");
+    }
+  }
+
+  _clearDropIndicators() {
+    this._dragList.querySelectorAll(".drop-indicator").forEach((el) => {
+      el.className = "drop-indicator";
+    });
+  }
+
+  _updateAutoScroll(clientY) {
+    const rect = this._listWrap.getBoundingClientRect();
+    const relY = clientY - rect.top;
+    const distTop = relY;
+    const distBot = rect.height - relY;
+
+    const inTop = distTop < this._SCROLL_ZONE && distTop >= 0;
+    const inBot = distBot < this._SCROLL_ZONE && distBot >= 0;
+
+    this._pulseTop.classList.toggle("active", inTop);
+    this._pulseBot.classList.toggle("active", inBot);
+
+    if ((inTop || inBot) && !this._autoScrollRAF) {
+      let lastTime = null;
+      const loop = (ts) => {
+        if (!this._ghost) {
+          this._stopAutoScroll();
+          return;
+        }
+        if (lastTime !== null) {
+          const dt = ts - lastTime;
+          const rect2 = this._listWrap.getBoundingClientRect();
+          const relY2 = this._lastPointerY - rect2.top;
+          const dTop = relY2;
+          const dBot = rect2.height - relY2;
+
+          if (dTop < this._SCROLL_ZONE && dTop >= 0) {
+            const factor = 1 - dTop / this._SCROLL_ZONE;
+            this._listWrap.scrollTop -= this._SCROLL_SPEED * dt * factor * 2.5;
+          } else if (dBot < this._SCROLL_ZONE && dBot >= 0) {
+            const factor = 1 - dBot / this._SCROLL_ZONE;
+            this._listWrap.scrollTop += this._SCROLL_SPEED * dt * factor * 2.5;
+          } else {
+            this._stopAutoScroll();
+            return;
+          }
+          this._updateDropTarget(this._lastPointerY);
+        }
+        lastTime = ts;
+        this._autoScrollRAF = requestAnimationFrame(loop);
+      };
+      this._autoScrollRAF = requestAnimationFrame(loop);
+    }
+
+    if (!inTop && !inBot) this._stopAutoScroll();
+  }
+
+  _stopAutoScroll() {
+    if (this._autoScrollRAF) {
+      cancelAnimationFrame(this._autoScrollRAF);
+      this._autoScrollRAF = null;
+    }
+    this._pulseTop.classList.remove("active");
+    this._pulseBot.classList.remove("active");
+  }
+}
+
+customElements.define("column-chooser", ColumnChooser);

datasette/static/datasette-manager.js

@@ -0,0 +1,222 @@
+// Custom events for use with the native CustomEvent API
+const DATASETTE_EVENTS = {
+  INIT: "datasette_init", // returns datasette manager instance in evt.detail
+};
+
+// Datasette "core" -> Methods/APIs that are foundational
+// Plugins will have greater stability if they use the functional hooks- but if they do decide to hook into
+// literal DOM selectors, they'll have an easier time using these addresses.
+const DOM_SELECTORS = {
+  /** Should have one match */
+  jsonExportLink: ".export-links a[href*=json]",
+
+  /** Event listeners that go outside of the main table, e.g. existing scroll listener */
+  tableWrapper: ".table-wrapper",
+  table: "table.rows-and-columns",
+  aboveTablePanel: ".above-table-panel",
+
+  // These could have multiple matches
+  /** Used for selecting table headers. Use makeColumnActions if you want to add menu items. */
+  tableHeaders: `table.rows-and-columns th`,
+
+  /** Used to add "where"  clauses to query using direct manipulation */
+  filterRows: ".filter-row",
+  /** Used to show top available enum values for a column ("facets") */
+  facetResults: ".facet-results [data-column]",
+};
+
+/**
+ * Monolith class for interacting with Datasette JS API
+ * Imported with DEFER, runs after main document parsed
+ * For now, manually synced with datasette/version.py
+ */
+const datasetteManager = {
+  VERSION: window.datasetteVersion,
+
+  // TODO: Should order of registration matter more?
+
+  // Should plugins be allowed to clobber others or is it last-in takes priority?
+  // Does pluginMetadata need to be serializable, or can we let it be stateful / have functions?
+  plugins: new Map(),
+
+  registerPlugin: (name, pluginMetadata) => {
+    if (datasetteManager.plugins.has(name)) {
+      console.warn(`Warning -> plugin ${name} was redefined`);
+    }
+    datasetteManager.plugins.set(name, pluginMetadata);
+
+    // If the plugin participates in the panel... update the panel.
+    if (pluginMetadata.makeAboveTablePanelConfigs) {
+      datasetteManager.renderAboveTablePanel();
+    }
+  },
+
+  /**
+   * New DOM elements are created on each click, so the data is not stale.
+   *
+   * Items
+   *  - must provide label (text)
+   *  - might provide href (string) or an onclick ((evt) => void)
+   *
+   * columnMeta is metadata stored on the column header (TH) as a DOMStringMap
+   * - column: string
+   * - columnNotNull: boolean
+   * - columnType: sqlite datatype enum (text, number, etc)
+   * - isPk: boolean
+   */
+  makeColumnActions: (columnMeta) => {
+    let columnActions = [];
+
+    // Accept function that returns list of columnActions with keys
+    // Required: label (text)
+    // Optional: onClick or href
+    datasetteManager.plugins.forEach((plugin) => {
+      if (plugin.makeColumnActions) {
+        // Plugins can provide multiple columnActions if they want
+        // If multiple try to create entry with same label, the last one deletes the others
+        columnActions.push(...plugin.makeColumnActions(columnMeta));
+      }
+    });
+
+    // TODO: Validate columnAction configs and give informative error message if missing keys.
+    return columnActions;
+  },
+
+  makeJumpSections: (context) => {
+    let jumpSections = [];
+
+    datasetteManager.plugins.forEach((plugin) => {
+      if (plugin.makeJumpSections) {
+        const sections = plugin.makeJumpSections(context) || [];
+        jumpSections.push(...sections);
+      }
+    });
+
+    return jumpSections;
+  },
+
+  /**
+   * In MVP, each plugin can only have 1 instance.
+   * In future, panels could be repeated. We omit that for now since so many plugins depend on
+   * shared URL state, so having multiple instances of plugin at same time is problematic.
+   * Currently, we never destroy any panels, we just hide them.
+   *
+   * TODO: nicer panel css, show panel selection state.
+   * TODO: does this hook need to take any arguments?
+   */
+  renderAboveTablePanel: () => {
+    const aboveTablePanel = document.querySelector(
+      DOM_SELECTORS.aboveTablePanel,
+    );
+
+    if (!aboveTablePanel) {
+      console.warn(
+        "This page does not have a table, the renderAboveTablePanel cannot be used.",
+      );
+      return;
+    }
+
+    let aboveTablePanelWrapper = aboveTablePanel.querySelector(".panels");
+
+    // First render: create wrappers. Otherwise, reuse previous.
+    if (!aboveTablePanelWrapper) {
+      aboveTablePanelWrapper = document.createElement("div");
+      aboveTablePanelWrapper.classList.add("tab-contents");
+      const panelNav = document.createElement("div");
+      panelNav.classList.add("tab-controls");
+
+      // Temporary: css for minimal amount of breathing room.
+      panelNav.style.display = "flex";
+      panelNav.style.gap = "8px";
+      panelNav.style.marginTop = "4px";
+      panelNav.style.marginBottom = "20px";
+
+      aboveTablePanel.appendChild(panelNav);
+      aboveTablePanel.appendChild(aboveTablePanelWrapper);
+    }
+
+    datasetteManager.plugins.forEach((plugin, pluginName) => {
+      const { makeAboveTablePanelConfigs } = plugin;
+
+      if (makeAboveTablePanelConfigs) {
+        const controls = aboveTablePanel.querySelector(".tab-controls");
+        const contents = aboveTablePanel.querySelector(".tab-contents");
+
+        // Each plugin can make multiple panels
+        const configs = makeAboveTablePanelConfigs();
+
+        configs.forEach((config, i) => {
+          const nodeContentId = `${pluginName}_${config.id}_panel-content`;
+
+          // quit if we've already registered this plugin
+          // TODO: look into whether plugins should be allowed to ask
+          // parent to re-render, or if they should manage that internally.
+          if (document.getElementById(nodeContentId)) {
+            return;
+          }
+
+          // Add tab control button
+          const pluginControl = document.createElement("button");
+          pluginControl.textContent = config.label;
+          pluginControl.onclick = () => {
+            contents.childNodes.forEach((node) => {
+              if (node.id === nodeContentId) {
+                node.style.display = "block";
+              } else {
+                node.style.display = "none";
+              }
+            });
+          };
+          controls.appendChild(pluginControl);
+
+          // Add plugin content area
+          const pluginNode = document.createElement("div");
+          pluginNode.id = nodeContentId;
+          config.render(pluginNode);
+          pluginNode.style.display = "none"; // Default to hidden unless you're ifrst
+
+          contents.appendChild(pluginNode);
+        });
+
+        // Let first node be selected by default
+        if (contents.childNodes.length) {
+          contents.childNodes[0].style.display = "block";
+        }
+      }
+    });
+  },
+
+  /** Selectors for document (DOM) elements. Store identifier instead of immediate references in case they haven't loaded when Manager starts. */
+  selectors: DOM_SELECTORS,
+
+  // Future API ideas
+  // Fetch page's data in array, and cache so plugins could reuse it
+  // Provide knowledge of what datasette JS or server-side via traditional console autocomplete
+  // State helpers: URL params https://github.com/simonw/datasette/issues/1144 and localstorage
+  // UI Hooks: command + k, tab manager hook
+  // Should we notify plugins that have dependencies
+  // when all dependencies were fulfilled? (leaflet, codemirror, etc)
+  // https://github.com/simonw/datasette-leaflet -> this way
+  // multiple plugins can all request the same copy of leaflet.
+};
+
+const initializeDatasette = () => {
+  // Hide the global behind __ prefix. Ideally they should be listening for the
+  // DATASETTE_EVENTS.INIT event to avoid the habit of reading from the window.
+
+  window.__DATASETTE__ = datasetteManager;
+
+  const initDatasetteEvent = new CustomEvent(DATASETTE_EVENTS.INIT, {
+    detail: datasetteManager,
+  });
+
+  document.dispatchEvent(initDatasetteEvent);
+};
+
+/**
+ * Main function
+ * Fires AFTER the document has been parsed
+ */
+document.addEventListener("DOMContentLoaded", function () {
+  initializeDatasette();
+});

datasette/static/json-format-highlight-1.0.1.js

@@ -0,0 +1,56 @@
+/*
+https://github.com/luyilin/json-format-highlight
+From https://unpkg.com/json-format-highlight@1.0.1/dist/json-format-highlight.js
+MIT Licensed
+*/
+(function (global, factory) {
+  typeof exports === "object" && typeof module !== "undefined"
+    ? (module.exports = factory())
+    : typeof define === "function" && define.amd
+      ? define(factory)
+      : (global.jsonFormatHighlight = factory());
+})(this, function () {
+  "use strict";
+
+  var defaultColors = {
+    keyColor: "dimgray",
+    numberColor: "lightskyblue",
+    stringColor: "lightcoral",
+    trueColor: "lightseagreen",
+    falseColor: "#f66578",
+    nullColor: "cornflowerblue",
+  };
+
+  function index(json, colorOptions) {
+    if (colorOptions === void 0) colorOptions = {};
+
+    if (!json) {
+      return;
+    }
+    if (typeof json !== "string") {
+      json = JSON.stringify(json, null, 2);
+    }
+    var colors = Object.assign({}, defaultColors, colorOptions);
+    json = json.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">");
+    return json.replace(
+      /("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+]?\d+)?)/g,
+      function (match) {
+        var color = colors.numberColor;
+        if (/^"/.test(match)) {
+          color = /:$/.test(match) ? colors.keyColor : colors.stringColor;
+        } else {
+          color = /true/.test(match)
+            ? colors.trueColor
+            : /false/.test(match)
+              ? colors.falseColor
+              : /null/.test(match)
+                ? colors.nullColor
+                : color;
+        }
+        return '<span style="color: ' + color + '">' + match + "</span>";
+      },
+    );
+  }
+
+  return index;
+});

datasette/static/mobile-column-actions.js

@@ -0,0 +1,318 @@
+var MOBILE_COLUMN_BREAKPOINT = 576;
+var MOBILE_COLUMN_DIALOG_ID = "mobile-column-actions-dialog";
+var MOBILE_COLUMN_DIALOG_TITLE_ID = "mobile-column-actions-title";
+
+function mobileColumnHeaders(manager) {
+  return Array.from(
+    document.querySelectorAll(manager.selectors.tableHeaders),
+  ).filter((th) => th.dataset.column && th.dataset.isLinkColumn !== "1");
+}
+
+function mobileColumnMetaText(th) {
+  var parts = [];
+  if (th.dataset.columnType) {
+    parts.push(th.dataset.columnType);
+  }
+  if (th.dataset.isPk === "1") {
+    parts.push("pk");
+  }
+  if (th.dataset.columnNotNull === "1") {
+    parts.push("not null");
+  }
+  return parts.join(", ");
+}
+
+function createMobileColumnActionNode(itemConfig, closeDialog) {
+  var actionNode;
+  if (itemConfig.href) {
+    actionNode = document.createElement("a");
+    actionNode.href = itemConfig.href;
+  } else {
+    actionNode = document.createElement("button");
+    actionNode.type = "button";
+  }
+  actionNode.textContent = itemConfig.label;
+
+  if (itemConfig.onClick) {
+    actionNode.addEventListener("click", function (ev) {
+      try {
+        itemConfig.onClick.call(actionNode, ev);
+      } finally {
+        closeDialog({ restoreFocus: false });
+      }
+    });
+  }
+
+  return actionNode;
+}
+
+function initMobileColumnActions(manager) {
+  var triggerButton = document.querySelector(".column-actions-mobile");
+  if (!triggerButton) {
+    return;
+  }
+
+  if (
+    !window.URLSearchParams ||
+    !window.HTMLDialogElement ||
+    !manager.columnActions
+  ) {
+    triggerButton.style.display = "none";
+    return;
+  }
+
+  if (!mobileColumnHeaders(manager).length) {
+    triggerButton.style.display = "none";
+    return;
+  }
+
+  var dialog = document.createElement("dialog");
+  dialog.className = "mobile-column-actions-dialog";
+  dialog.id = MOBILE_COLUMN_DIALOG_ID;
+  dialog.setAttribute("aria-labelledby", MOBILE_COLUMN_DIALOG_TITLE_ID);
+  dialog.innerHTML = `
+    <div class="modal-header">
+      <span class="modal-title" id="${MOBILE_COLUMN_DIALOG_TITLE_ID}">Column actions</span>
+      <span class="modal-meta"></span>
+    </div>
+    <div class="list-wrap mobile-column-list"></div>
+    <div class="modal-footer">
+      <span class="footer-info">Tap a column to reveal actions.</span>
+      <button type="button" class="btn btn-ghost mobile-column-actions-done">Done</button>
+    </div>
+  `;
+  document.body.appendChild(dialog);
+
+  triggerButton.setAttribute("aria-haspopup", "dialog");
+  triggerButton.setAttribute("aria-controls", MOBILE_COLUMN_DIALOG_ID);
+  triggerButton.setAttribute("aria-expanded", "false");
+
+  var countEl = dialog.querySelector(".modal-meta");
+  var listWrap = dialog.querySelector(".mobile-column-list");
+  var doneButton = dialog.querySelector(".mobile-column-actions-done");
+  var expandedSectionId = null;
+  var shouldRestoreFocus = true;
+
+  function updateExpandedSection() {
+    Array.from(dialog.querySelectorAll(".col-header")).forEach((button) => {
+      var controlsId = button.getAttribute("aria-controls");
+      var actionList = dialog.querySelector("#" + controlsId);
+      var isExpanded = controlsId === expandedSectionId;
+      button.setAttribute("aria-expanded", isExpanded ? "true" : "false");
+      actionList.hidden = !isExpanded;
+      actionList.classList.toggle("expanded", isExpanded);
+    });
+  }
+
+  function scrollExpandedSectionIntoView(section) {
+    var sectionTop = section.offsetTop;
+    var sectionBottom = sectionTop + section.offsetHeight;
+    var visibleTop = listWrap.scrollTop;
+    var visibleBottom = visibleTop + listWrap.clientHeight;
+    var sectionHeight = section.offsetHeight;
+
+    if (sectionTop < visibleTop) {
+      listWrap.scrollTop = sectionTop;
+      return;
+    }
+
+    if (sectionBottom <= visibleBottom) {
+      return;
+    }
+
+    if (sectionHeight <= listWrap.clientHeight) {
+      listWrap.scrollTop = sectionBottom - listWrap.clientHeight;
+    } else {
+      listWrap.scrollTop = sectionTop;
+    }
+  }
+
+  function closeDialog(options) {
+    options = options || {};
+    shouldRestoreFocus = options.restoreFocus !== false;
+    if (dialog.open) {
+      dialog.close();
+    } else {
+      triggerButton.setAttribute("aria-expanded", "false");
+      if (shouldRestoreFocus) {
+        triggerButton.focus();
+      }
+    }
+  }
+
+  function renderDialog() {
+    var headers = mobileColumnHeaders(manager);
+    if (!headers.length) {
+      closeDialog({ restoreFocus: false });
+      triggerButton.style.display = "none";
+      return false;
+    }
+
+    if (
+      !headers.some(
+        (_th, index) => `mobile-column-actions-${index}` === expandedSectionId,
+      )
+    ) {
+      expandedSectionId = null;
+    }
+
+    countEl.textContent = `${headers.length} column${
+      headers.length === 1 ? "" : "s"
+    }`;
+    listWrap.innerHTML = "";
+
+    if (manager.columnActions.shouldShowShowAllColumns()) {
+      var topActions = document.createElement("div");
+      topActions.className = "mobile-column-top-actions";
+
+      var showAllColumns = document.createElement("a");
+      showAllColumns.className = "btn btn-ghost mobile-column-top-action";
+      showAllColumns.href = manager.columnActions.showAllColumnsUrl();
+      showAllColumns.textContent = "Show all columns";
+
+      topActions.appendChild(showAllColumns);
+      listWrap.appendChild(topActions);
+    }
+
+    headers.forEach((th, index) => {
+      var sectionId = `mobile-column-actions-${index}`;
+      var actionState = manager.columnActions.buildColumnActionState(th, {
+        includeChooseColumns: false,
+        includeShowAllColumns: false,
+      });
+      var section = document.createElement("section");
+      section.className = "mobile-column-section";
+
+      var headerButton = document.createElement("button");
+      headerButton.type = "button";
+      headerButton.className = "col-header";
+      headerButton.setAttribute("aria-controls", sectionId);
+      headerButton.setAttribute("aria-expanded", "false");
+
+      var headerText = document.createElement("span");
+      headerText.className = "mobile-column-header-text";
+
+      var name = document.createElement("span");
+      name.className = "mobile-column-name";
+      name.textContent = th.dataset.column;
+      headerText.appendChild(name);
+
+      var metaText = mobileColumnMetaText(th);
+      if (metaText) {
+        var meta = document.createElement("span");
+        meta.className = "mobile-column-meta";
+        meta.textContent = metaText;
+        headerText.appendChild(meta);
+      }
+
+      var chevron = document.createElement("span");
+      chevron.className = "mobile-column-chevron";
+      chevron.setAttribute("aria-hidden", "true");
+      chevron.textContent = "▾";
+
+      headerButton.appendChild(headerText);
+      headerButton.appendChild(chevron);
+      headerButton.addEventListener("click", function () {
+        expandedSectionId = expandedSectionId === sectionId ? null : sectionId;
+        updateExpandedSection();
+        if (expandedSectionId === sectionId) {
+          scrollExpandedSectionIntoView(section);
+        }
+      });
+
+      var actionContainer = document.createElement("div");
+      actionContainer.id = sectionId;
+      actionContainer.className = "col-actions";
+      actionContainer.hidden = true;
+
+      if (actionState.columnDescription) {
+        var description = document.createElement("p");
+        description.className = "mobile-column-description";
+        description.textContent = actionState.columnDescription;
+        actionContainer.appendChild(description);
+      }
+
+      if (actionState.actionItems.length) {
+        var actionList = document.createElement("ul");
+        actionState.actionItems.forEach((itemConfig) => {
+          var actionItem = document.createElement("li");
+          actionItem.appendChild(
+            createMobileColumnActionNode(itemConfig, closeDialog),
+          );
+          actionList.appendChild(actionItem);
+        });
+        actionContainer.appendChild(actionList);
+      } else {
+        var noActions = document.createElement("p");
+        noActions.className = "mobile-column-no-actions";
+        noActions.textContent = "No actions available";
+        actionContainer.appendChild(noActions);
+      }
+
+      section.appendChild(headerButton);
+      section.appendChild(actionContainer);
+      listWrap.appendChild(section);
+    });
+
+    updateExpandedSection();
+    return true;
+  }
+
+  function openDialog() {
+    if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT) {
+      return;
+    }
+    if (!renderDialog()) {
+      return;
+    }
+    if (!dialog.open) {
+      dialog.showModal();
+    }
+    triggerButton.setAttribute("aria-expanded", "true");
+    var focusTarget =
+      dialog.querySelector(".mobile-column-top-action") ||
+      dialog.querySelector(".col-header") ||
+      doneButton;
+    focusTarget.focus();
+  }
+
+  triggerButton.addEventListener("click", function () {
+    if (dialog.open) {
+      closeDialog();
+    } else {
+      openDialog();
+    }
+  });
+
+  doneButton.addEventListener("click", function () {
+    closeDialog();
+  });
+
+  dialog.addEventListener("click", function (ev) {
+    if (ev.target === dialog) {
+      closeDialog();
+    }
+  });
+
+  dialog.addEventListener("cancel", function (ev) {
+    ev.preventDefault();
+    closeDialog();
+  });
+
+  dialog.addEventListener("close", function () {
+    triggerButton.setAttribute("aria-expanded", "false");
+    if (shouldRestoreFocus) {
+      triggerButton.focus();
+    }
+  });
+
+  window.addEventListener("resize", function () {
+    if (window.innerWidth > MOBILE_COLUMN_BREAKPOINT && dialog.open) {
+      closeDialog({ restoreFocus: false });
+    }
+  });
+}
+
+document.addEventListener("datasette_init", function (evt) {
+  initMobileColumnActions(evt.detail);
+});

datasette/static/navigation-search.js

@@ -0,0 +1,910 @@
+let navigationSearchInstanceCounter = 0;
+
+class NavigationSearch extends HTMLElement {
+  constructor() {
+    super();
+    this.instanceId = ++navigationSearchInstanceCounter;
+    this.inputId = `navigation-search-input-${this.instanceId}`;
+    this.instructionsId = `navigation-search-instructions-${this.instanceId}`;
+    this.listboxId = `navigation-search-results-${this.instanceId}`;
+    this.recentHeadingId = `navigation-search-recent-${this.instanceId}`;
+    this.statusId = `navigation-search-status-${this.instanceId}`;
+    this.titleId = `navigation-search-title-${this.instanceId}`;
+    this.attachShadow({ mode: "open" });
+    this.selectedIndex = -1;
+    this.matches = [];
+    this.renderedMatches = [];
+    this.debounceTimer = null;
+    this.restoreFocusTarget = null;
+    this.shouldRestoreFocus = true;
+
+    this.render();
+    this.setupEventListeners();
+  }
+
+  render() {
+    this.shadowRoot.innerHTML = `
+            <style>
+                :host {
+                    display: contents;
+                }
+
+                dialog {
+                    border: none;
+                    border-radius: var(--modal-border-radius, 0.75rem);
+                    padding: 0;
+                    max-width: 90vw;
+                    width: 600px;
+                    max-height: 80vh;
+                    box-shadow: var(--modal-shadow, 0 20px 25px -5px rgba(0, 0, 0, 0.1), 0 10px 10px -5px rgba(0, 0, 0, 0.04));
+                    animation: slideIn var(--modal-animation-duration, 0.2s) ease-out;
+                }
+
+                dialog::backdrop {
+                    background: var(--modal-backdrop-bg, rgba(0, 0, 0, 0.5));
+                    backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+                    -webkit-backdrop-filter: var(--modal-backdrop-blur, blur(4px));
+                    animation: fadeIn var(--modal-animation-duration, 0.2s) ease-out;
+                }
+
+                @keyframes slideIn {
+                    from {
+                        opacity: 0;
+                        transform: translateY(-20px) scale(0.95);
+                    }
+                    to {
+                        opacity: 1;
+                        transform: translateY(0) scale(1);
+                    }
+                }
+
+                @keyframes fadeIn {
+                    from { opacity: 0; }
+                    to { opacity: 1; }
+                }
+
+                .search-container {
+                    display: flex;
+                    flex-direction: column;
+                }
+
+                .search-input-wrapper {
+                    padding: 1.25rem;
+                    border-bottom: 1px solid #e5e7eb;
+                    display: flex;
+                    gap: 0.5rem;
+                    align-items: center;
+                }
+
+                .search-input {
+                    width: 100%;
+                    flex: 1;
+                    min-width: 0;
+                    padding: 0.75rem 1rem;
+                    font-size: 1rem;
+                    border: 2px solid #e5e7eb;
+                    border-radius: 0.5rem;
+                    outline: none;
+                    transition: border-color 0.2s;
+                    box-sizing: border-box;
+                }
+
+                .search-input:focus {
+                    border-color: #2563eb;
+                }
+
+                .close-search {
+                    background: transparent;
+                    border: 1px solid transparent;
+                    border-radius: 0.375rem;
+                    color: #4b5563;
+                    cursor: pointer;
+                    flex: 0 0 auto;
+                    font: inherit;
+                    font-size: 1.5rem;
+                    height: 2.75rem;
+                    line-height: 1;
+                    width: 2.75rem;
+                }
+
+                .close-search:hover,
+                .close-search:focus {
+                    background-color: #f3f4f6;
+                    border-color: #d1d5db;
+                }
+
+                .results-container {
+                    overflow-y: auto;
+                    height: calc(80vh - 180px);
+                    padding: 0.5rem;
+                }
+
+                .results-list:empty {
+                    display: none;
+                }
+
+                .result-item {
+                    padding: 0.875rem 1rem;
+                    cursor: pointer;
+                    border-radius: 0.5rem;
+                    transition: background-color 0.15s;
+                    display: flex;
+                    align-items: center;
+                    gap: 0.75rem;
+                }
+
+                .result-item:hover {
+                    background-color: #f3f4f6;
+                }
+
+                .result-item.selected {
+                    background-color: #dbeafe;
+                }
+
+                .result-item > div {
+                    flex: 1;
+                    min-width: 0;
+                }
+
+                .jump-start-content {
+                    border-bottom: 1px solid #e5e7eb;
+                    margin-bottom: 0.5rem;
+                    padding: 0.5rem 0.5rem 1rem;
+                }
+
+                .jump-start-content:empty {
+                    display: none;
+                }
+
+                .result-name {
+                    font-weight: 500;
+                    color: #111827;
+                }
+
+                .result-label {
+                    font-size: 0.875rem;
+                    color: #4b5563;
+                }
+
+                .result-type {
+                    color: #4b5563;
+                    font-size: 0.75rem;
+                    font-weight: 600;
+                    text-transform: uppercase;
+                }
+
+                .result-url {
+                    font-size: 0.875rem;
+                    color: #6b7280;
+                }
+
+                .result-description {
+                    color: #374151;
+                    display: -webkit-box;
+                    font-size: 0.8125rem;
+                    line-height: 1.35;
+                    margin-top: 0.35rem;
+                    overflow: hidden;
+                    -webkit-box-orient: vertical;
+                    -webkit-line-clamp: 2;
+                }
+
+                .results-heading {
+                    color: #4b5563;
+                    font-size: 0.75rem;
+                    font-weight: 600;
+                    letter-spacing: 0;
+                    padding: 0.5rem 1rem 0.25rem;
+                    text-transform: uppercase;
+                }
+
+                .recent-actions {
+                    padding: 0.25rem 1rem 0.75rem;
+                }
+
+                .clear-recent {
+                    background: transparent;
+                    border: 0;
+                    color: #2563eb;
+                    cursor: pointer;
+                    font: inherit;
+                    font-size: 0.875rem;
+                    padding: 0;
+                }
+
+                .clear-recent:hover {
+                    text-decoration: underline;
+                }
+
+                .no-results {
+                    padding: 2rem;
+                    text-align: center;
+                    color: #6b7280;
+                }
+
+                .hint-text {
+                    padding: 0.75rem 1.25rem;
+                    font-size: 0.875rem;
+                    color: #6b7280;
+                    border-top: 1px solid #e5e7eb;
+                    display: flex;
+                    gap: 1rem;
+                    flex-wrap: wrap;
+                }
+
+                .hint-text kbd {
+                    background: #f3f4f6;
+                    padding: 0.125rem 0.375rem;
+                    border-radius: 0.25rem;
+                    font-size: 0.75rem;
+                    border: 1px solid #d1d5db;
+                    font-family: monospace;
+                }
+
+                .visually-hidden {
+                    border: 0;
+                    clip: rect(0 0 0 0);
+                    height: 1px;
+                    margin: -1px;
+                    overflow: hidden;
+                    padding: 0;
+                    position: absolute;
+                    white-space: nowrap;
+                    width: 1px;
+                }
+
+                /* Mobile optimizations */
+                @media (max-width: 640px) {
+                    dialog {
+                        width: 95vw;
+                        max-height: 85vh;
+                        border-radius: 0.5rem;
+                    }
+
+                    .search-input-wrapper {
+                        padding: 1rem;
+                    }
+
+                    .search-input {
+                        font-size: 16px; /* Prevents zoom on iOS */
+                    }
+
+                    .result-item {
+                        padding: 1rem 0.75rem;
+                    }
+
+                    .hint-text {
+                        font-size: 0.8rem;
+                        padding: 0.5rem 1rem;
+                    }
+                }
+            </style>
+
+            <dialog aria-modal="true" aria-labelledby="${this.titleId}">
+                <div class="search-container">
+                    <h2 id="${this.titleId}" class="visually-hidden">Jump to</h2>
+                    <p id="${this.instructionsId}" class="visually-hidden">Type to search. Use up and down arrow keys to move through results, Enter to select a result, and Escape to close this menu.</p>
+                    <div id="${this.statusId}" class="visually-hidden" aria-live="polite" aria-atomic="true"></div>
+                    <div class="search-input-wrapper">
+                        <input 
+                            id="${this.inputId}"
+                            type="text" 
+                            class="search-input" 
+                            placeholder="Jump to..."
+                            aria-label="Jump to"
+                            aria-describedby="${this.instructionsId}"
+                            role="combobox"
+                            aria-autocomplete="list"
+                            aria-controls="${this.listboxId}"
+                            aria-expanded="false"
+                            autocomplete="off"
+                            spellcheck="false"
+                        >
+                        <button type="button" class="close-search" aria-label="Close jump menu">&times;</button>
+                    </div>
+                    <div class="results-container"></div>
+                    <div class="hint-text">
+                        <span><kbd>↑</kbd> <kbd>↓</kbd> Navigate</span>
+                        <span><kbd>Enter</kbd> Select</span>
+                        <span><kbd>Esc</kbd> Close</span>
+                    </div>
+                </div>
+            </dialog>
+        `;
+  }
+
+  setupEventListeners() {
+    const dialog = this.shadowRoot.querySelector("dialog");
+    const input = this.shadowRoot.querySelector(".search-input");
+    const closeButton = this.shadowRoot.querySelector(".close-search");
+    const resultsContainer =
+      this.shadowRoot.querySelector(".results-container");
+
+    // Global keyboard listener for "/"
+    document.addEventListener("keydown", (e) => {
+      if (e.key === "/" && !this.isInputFocused() && !dialog.open) {
+        e.preventDefault();
+        this.openMenu();
+      }
+    });
+
+    document.addEventListener("click", (e) => {
+      const trigger = e.target.closest("[data-navigation-search-open]");
+      if (trigger) {
+        e.preventDefault();
+        const details = trigger.closest("details");
+        const restoreTarget = details?.querySelector("summary") || trigger;
+        details?.removeAttribute("open");
+        this.openMenu(restoreTarget);
+      }
+    });
+
+    // Input event
+    input.addEventListener("input", (e) => {
+      this.handleSearch(e.target.value);
+    });
+
+    // Keyboard navigation
+    input.addEventListener("keydown", (e) => {
+      if (e.key === "ArrowDown") {
+        e.preventDefault();
+        this.moveSelection(1);
+      } else if (e.key === "ArrowUp") {
+        e.preventDefault();
+        this.moveSelection(-1);
+      } else if (e.key === "Enter") {
+        e.preventDefault();
+        this.selectCurrentItem();
+      } else if (e.key === "Escape") {
+        this.closeMenu();
+      }
+    });
+
+    closeButton.addEventListener("click", () => {
+      this.closeMenu();
+    });
+
+    // Click on result item
+    resultsContainer.addEventListener("click", (e) => {
+      const clearRecent = e.target.closest("[data-clear-recent-items]");
+      if (clearRecent) {
+        e.preventDefault();
+        this.clearRecentItems();
+        return;
+      }
+
+      const item = e.target.closest(".result-item");
+      if (item) {
+        const index = parseInt(item.dataset.index);
+        this.selectItem(index);
+      }
+    });
+
+    // Close on backdrop click
+    dialog.addEventListener("click", (e) => {
+      if (e.target === dialog) {
+        this.closeMenu();
+      }
+    });
+
+    dialog.addEventListener("cancel", (e) => {
+      e.preventDefault();
+      this.closeMenu();
+    });
+
+    dialog.addEventListener("close", () => {
+      this.onMenuClosed();
+    });
+
+    // Initial load
+    this.loadInitialData();
+  }
+
+  isInputFocused() {
+    const activeElement = document.activeElement;
+    return (
+      activeElement &&
+      (activeElement.tagName === "INPUT" ||
+        activeElement.tagName === "TEXTAREA" ||
+        activeElement.isContentEditable)
+    );
+  }
+
+  setElementAttribute(element, name, value) {
+    if (!element) {
+      return;
+    }
+    if (typeof element.setAttribute === "function") {
+      element.setAttribute(name, value);
+    } else {
+      element[name] = String(value);
+    }
+  }
+
+  removeElementAttribute(element, name) {
+    if (!element) {
+      return;
+    }
+    if (typeof element.removeAttribute === "function") {
+      element.removeAttribute(name);
+    } else {
+      delete element[name];
+    }
+  }
+
+  focusRestoreTarget(trigger) {
+    if (trigger && typeof trigger.focus === "function") {
+      return trigger;
+    }
+    if (
+      document.activeElement &&
+      typeof document.activeElement.focus === "function"
+    ) {
+      return document.activeElement;
+    }
+    return null;
+  }
+
+  setNavigationTriggersExpanded(expanded) {
+    if (typeof document.querySelectorAll !== "function") {
+      return;
+    }
+    document
+      .querySelectorAll("[data-navigation-search-open]")
+      .forEach((trigger) => {
+        this.setElementAttribute(
+          trigger,
+          "aria-expanded",
+          expanded ? "true" : "false",
+        );
+      });
+  }
+
+  resultOptionId(index) {
+    return `${this.listboxId}-option-${index}`;
+  }
+
+  updateComboboxState() {
+    const dialog = this.shadowRoot.querySelector("dialog");
+    const input = this.shadowRoot.querySelector(".search-input");
+    const matches = this.renderedMatches || [];
+    this.setElementAttribute(
+      input,
+      "aria-expanded",
+      dialog && dialog.open && matches.length > 0 ? "true" : "false",
+    );
+
+    if (
+      dialog &&
+      dialog.open &&
+      this.selectedIndex >= 0 &&
+      this.selectedIndex < matches.length
+    ) {
+      this.setElementAttribute(
+        input,
+        "aria-activedescendant",
+        this.resultOptionId(this.selectedIndex),
+      );
+    } else {
+      this.removeElementAttribute(input, "aria-activedescendant");
+    }
+  }
+
+  setStatus(message) {
+    const status = this.shadowRoot.querySelector(`#${this.statusId}`);
+    if (status) {
+      status.textContent = message || "";
+    }
+  }
+
+  resultsStatus(count, truncated) {
+    if (truncated) {
+      return "More than 100 results. Keep typing to narrow the list.";
+    }
+    if (count === 0) {
+      return "No results found.";
+    }
+    if (count === 1) {
+      return "1 result.";
+    }
+    return `${count} results.`;
+  }
+
+  loadInitialData() {
+    const itemsAttr = this.getAttribute("items");
+    if (itemsAttr) {
+      try {
+        this.allItems = JSON.parse(itemsAttr);
+        this.matches = this.allItems;
+      } catch (e) {
+        console.error("Failed to parse items attribute:", e);
+        this.allItems = [];
+        this.matches = [];
+      }
+    }
+  }
+
+  handleSearch(query) {
+    clearTimeout(this.debounceTimer);
+    if (query.trim()) {
+      this.setStatus("Searching...");
+    } else {
+      this.setStatus("");
+    }
+
+    this.debounceTimer = setTimeout(() => {
+      const url = this.getAttribute("url");
+
+      if (url) {
+        // Fetch from API
+        this.fetchResults(url, query);
+      } else {
+        // Filter local items
+        this.filterLocalItems(query);
+      }
+    }, 200);
+  }
+
+  async fetchResults(url, query) {
+    try {
+      const searchUrl = `${url}?q=${encodeURIComponent(query)}`;
+      const response = await fetch(searchUrl);
+      const data = await response.json();
+      this.matches = data.matches || [];
+      this.selectedIndex = this.matches.length > 0 ? 0 : -1;
+      this.renderResults();
+      if (query.trim()) {
+        this.setStatus(this.resultsStatus(this.matches.length, data.truncated));
+      } else {
+        this.setStatus("");
+      }
+    } catch (e) {
+      console.error("Failed to fetch search results:", e);
+      this.matches = [];
+      this.renderResults();
+      this.setStatus("Search failed.");
+    }
+  }
+
+  filterLocalItems(query) {
+    if (!query.trim()) {
+      this.matches = this.allItems || [];
+    } else {
+      const lowerQuery = query.toLowerCase();
+      this.matches = (this.allItems || []).filter(
+        (item) =>
+          item.name.toLowerCase().includes(lowerQuery) ||
+          (item.display_name || "").toLowerCase().includes(lowerQuery) ||
+          item.url.toLowerCase().includes(lowerQuery),
+      );
+    }
+    this.selectedIndex = this.matches.length > 0 ? 0 : -1;
+    this.renderResults();
+    if (query.trim()) {
+      this.setStatus(this.resultsStatus(this.matches.length, false));
+    } else {
+      this.setStatus("");
+    }
+  }
+
+  recentItemsStorageKey() {
+    return "datasette.navigationSearch.recentItems";
+  }
+
+  loadRecentItems() {
+    if (typeof localStorage === "undefined") {
+      return [];
+    }
+
+    try {
+      const raw = localStorage.getItem(this.recentItemsStorageKey());
+      if (!raw) {
+        return [];
+      }
+      const parsed = JSON.parse(raw);
+      if (!Array.isArray(parsed)) {
+        return [];
+      }
+      return parsed
+        .filter((item) => item && item.name && item.url)
+        .map((item) => ({
+          name: String(item.name),
+          display_name: item.display_name ? String(item.display_name) : "",
+          url: String(item.url),
+          type: item.type ? String(item.type) : "",
+          description: item.description ? String(item.description) : "",
+        }))
+        .slice(0, 5);
+    } catch (e) {
+      return [];
+    }
+  }
+
+  saveRecentItem(match) {
+    if (
+      typeof localStorage === "undefined" ||
+      !match ||
+      !match.name ||
+      !match.url
+    ) {
+      return;
+    }
+
+    try {
+      const item = {
+        name: String(match.name),
+        display_name: match.display_name ? String(match.display_name) : "",
+        url: String(match.url),
+        type: match.type ? String(match.type) : "",
+        description: match.description ? String(match.description) : "",
+      };
+      const recentItems = this.loadRecentItems().filter(
+        (recentItem) => recentItem.url !== item.url,
+      );
+      localStorage.setItem(
+        this.recentItemsStorageKey(),
+        JSON.stringify([item, ...recentItems].slice(0, 5)),
+      );
+    } catch (e) {
+      // localStorage may be unavailable, full, or disabled.
+    }
+  }
+
+  clearRecentItems() {
+    if (typeof localStorage === "undefined") {
+      return;
+    }
+
+    try {
+      localStorage.removeItem(this.recentItemsStorageKey());
+    } catch (e) {
+      localStorage.setItem(this.recentItemsStorageKey(), "[]");
+    }
+    this.renderResults();
+    this.setStatus("Recent items cleared.");
+  }
+
+  jumpSections() {
+    const manager = window.__DATASETTE__;
+    if (!manager || typeof manager.makeJumpSections !== "function") {
+      return [];
+    }
+    const sections = manager.makeJumpSections({
+      navigationSearch: this,
+    });
+    return Array.isArray(sections)
+      ? sections.filter(
+          (section) => section && typeof section.render === "function",
+        )
+      : [];
+  }
+
+  jumpSectionsHtml(jumpSections) {
+    return jumpSections
+      .map((section, index) => {
+        const id = section.id
+          ? ` data-jump-section-id="${this.escapeHtml(section.id)}"`
+          : "";
+        return `<div class="jump-start-content" data-jump-section-index="${index}"${id}></div>`;
+      })
+      .join("");
+  }
+
+  renderJumpSections(container, jumpSections) {
+    jumpSections.forEach((section, index) => {
+      const node = container.querySelector(
+        `[data-jump-section-index="${index}"]`,
+      );
+      if (!node) {
+        return;
+      }
+      section.render(node, {
+        navigationSearch: this,
+        container,
+        input: this.shadowRoot.querySelector(".search-input"),
+      });
+    });
+  }
+
+  resultItemHtml(match, index) {
+    const displayName = match.display_name || match.name;
+    const label =
+      match.display_name && match.display_name !== match.name
+        ? `<div class="result-label">${this.escapeHtml(match.name)}</div>`
+        : "";
+    const type = match.type
+      ? `<div class="result-type">${this.escapeHtml(match.type)}</div>`
+      : "";
+    const description = match.description
+      ? `<div class="result-description">${this.escapeHtml(
+          match.description,
+        )}</div>`
+      : "";
+    return `
+            <div
+                id="${this.resultOptionId(index)}"
+                class="result-item ${index === this.selectedIndex ? "selected" : ""}"
+                data-index="${index}"
+                role="option"
+                aria-selected="${index === this.selectedIndex}"
+            >
+                <div>
+                    ${type}
+                    <div class="result-name">${this.escapeHtml(displayName)}</div>
+                    ${label}
+                    <div class="result-url">${this.escapeHtml(match.url)}</div>
+                    ${description}
+                </div>
+            </div>
+        `;
+  }
+
+  renderResults() {
+    const container = this.shadowRoot.querySelector(".results-container");
+    const input = this.shadowRoot.querySelector(".search-input");
+    const showStartContent = !input.value.trim();
+    const jumpSections = showStartContent ? this.jumpSections() : [];
+    const startBlock = showStartContent
+      ? this.jumpSectionsHtml(jumpSections)
+      : "";
+    const recentItems = showStartContent ? this.loadRecentItems() : [];
+    const defaultMatches = showStartContent ? [] : this.matches;
+    const renderedMatches = [...recentItems, ...defaultMatches];
+    this.renderedMatches = renderedMatches;
+    const emptyListbox = `<div id="${this.listboxId}" class="results-list" role="listbox" aria-label="Jump results"></div>`;
+
+    if (renderedMatches.length) {
+      if (
+        this.selectedIndex < 0 ||
+        this.selectedIndex >= renderedMatches.length
+      ) {
+        this.selectedIndex = 0;
+      }
+    } else {
+      this.selectedIndex = -1;
+    }
+
+    if (renderedMatches.length === 0) {
+      if (startBlock) {
+        container.innerHTML = startBlock + emptyListbox;
+        this.renderJumpSections(container, jumpSections);
+      } else if (showStartContent) {
+        container.innerHTML = emptyListbox;
+      } else {
+        const message = input.value.trim()
+          ? "No results found"
+          : "Start typing to search...";
+        container.innerHTML = `${emptyListbox}<div class="no-results">${message}</div>`;
+      }
+      this.updateComboboxState();
+      return;
+    }
+
+    const recentHeading = recentItems.length
+      ? `<div class="results-heading" id="${this.recentHeadingId}">Recent</div>`
+      : "";
+    const recentGroup = recentItems.length
+      ? `<div role="group" aria-labelledby="${this.recentHeadingId}">${recentItems
+          .map((match, index) => this.resultItemHtml(match, index))
+          .join("")}</div>`
+      : "";
+    const recentActions = recentItems.length
+      ? `<div class="recent-actions"><button type="button" class="clear-recent" data-clear-recent-items>Clear recent</button></div>`
+      : "";
+    const defaultHtml = defaultMatches
+      .map((match, index) =>
+        this.resultItemHtml(match, recentItems.length + index),
+      )
+      .join("");
+    container.innerHTML =
+      startBlock +
+      recentHeading +
+      `<div id="${this.listboxId}" class="results-list" role="listbox" aria-label="Jump results">${recentGroup}${defaultHtml}</div>` +
+      recentActions;
+    this.renderJumpSections(container, jumpSections);
+    this.updateComboboxState();
+
+    // Scroll selected item into view
+    if (this.selectedIndex >= 0) {
+      const selectedItem = container.querySelector(
+        `.result-item[data-index="${this.selectedIndex}"]`,
+      );
+      if (selectedItem) {
+        selectedItem.scrollIntoView({ block: "nearest" });
+      }
+    }
+  }
+
+  moveSelection(direction) {
+    const matches = this.renderedMatches || this.matches;
+    const newIndex = this.selectedIndex + direction;
+    if (newIndex >= 0 && newIndex < matches.length) {
+      this.selectedIndex = newIndex;
+      this.renderResults();
+    }
+  }
+
+  selectCurrentItem() {
+    const matches = this.renderedMatches || this.matches;
+    if (this.selectedIndex >= 0 && this.selectedIndex < matches.length) {
+      this.selectItem(this.selectedIndex);
+    }
+  }
+
+  selectItem(index) {
+    const matches = this.renderedMatches || this.matches;
+    const match = matches[index];
+    if (match) {
+      this.saveRecentItem(match);
+
+      // Dispatch custom event
+      this.dispatchEvent(
+        new CustomEvent("select", {
+          detail: match,
+          bubbles: true,
+          composed: true,
+        }),
+      );
+
+      // Navigate to URL
+      window.location.href = match.url;
+
+      this.closeMenu({ restoreFocus: false });
+    }
+  }
+
+  openMenu(trigger) {
+    const dialog = this.shadowRoot.querySelector("dialog");
+    const input = this.shadowRoot.querySelector(".search-input");
+
+    this.restoreFocusTarget = this.focusRestoreTarget(trigger);
+    this.shouldRestoreFocus = true;
+    if (!dialog.open) {
+      dialog.showModal();
+    }
+    this.setNavigationTriggersExpanded(true);
+    input.value = "";
+    input.focus();
+
+    // Reset state, then populate the default jump list.
+    this.matches = [];
+    this.selectedIndex = -1;
+    this.renderResults();
+    this.setStatus("");
+  }
+
+  closeMenu(options = {}) {
+    const dialog = this.shadowRoot.querySelector("dialog");
+    this.shouldRestoreFocus = options.restoreFocus !== false;
+    if (dialog.open) {
+      dialog.close();
+    } else {
+      this.onMenuClosed();
+    }
+  }
+
+  onMenuClosed() {
+    const input = this.shadowRoot.querySelector(".search-input");
+    this.setElementAttribute(input, "aria-expanded", "false");
+    this.removeElementAttribute(input, "aria-activedescendant");
+    this.setNavigationTriggersExpanded(false);
+    this.setStatus("");
+    if (
+      this.shouldRestoreFocus &&
+      this.restoreFocusTarget &&
+      typeof this.restoreFocusTarget.focus === "function"
+    ) {
+      this.restoreFocusTarget.focus();
+    }
+    this.restoreFocusTarget = null;
+  }
+
+  escapeHtml(text) {
+    const div = document.createElement("div");
+    div.textContent = text == null ? "" : text;
+    return div.innerHTML;
+  }
+}
+
+// Register the custom element
+customElements.define("navigation-search", NavigationSearch);

datasette/static/table.js

@@ -0,0 +1,757 @@
+var DROPDOWN_HTML = `<div class="dropdown-menu">
+<div class="hook"></div>
+<ul class="dropdown-actions"></ul>
+<p class="dropdown-column-type"></p>
+<p class="dropdown-column-description"></p>
+</div>`;
+
+var DROPDOWN_ICON_SVG = `<svg xmlns="http://www.w3.org/2000/svg" width="14" height="14" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+  <circle cx="12" cy="12" r="3"></circle>
+  <path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1-2-2 2 2 0 0 1 2-2h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 2-2 2 2 0 0 1 2 2v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-.09a1.65 1.65 0 0 0-1.51 1z"></path>
+</svg>`;
+
+var SET_COLUMN_TYPE_DIALOG_ID = "set-column-type-dialog";
+var setColumnTypeDialogState = null;
+
+function getParams() {
+  return new URLSearchParams(location.search);
+}
+
+function paramsToUrl(params) {
+  var s = params.toString();
+  return s ? "?" + s : location.pathname;
+}
+
+function sortDescUrl(column) {
+  var params = getParams();
+  params.set("_sort_desc", column);
+  params.delete("_sort");
+  params.delete("_next");
+  return paramsToUrl(params);
+}
+
+function sortAscUrl(column) {
+  var params = getParams();
+  params.set("_sort", column);
+  params.delete("_sort_desc");
+  params.delete("_next");
+  return paramsToUrl(params);
+}
+
+function facetUrl(column) {
+  var params = getParams();
+  params.append("_facet", column);
+  return paramsToUrl(params);
+}
+
+function hideColumnUrl(column) {
+  var params = getParams();
+  params.append("_nocol", column);
+  return paramsToUrl(params);
+}
+
+function showAllColumnsUrl() {
+  var params = getParams();
+  params.delete("_nocol");
+  params.delete("_col");
+  return paramsToUrl(params);
+}
+
+function notBlankUrl(column) {
+  var params = getParams();
+  params.set(`${column}__notblank`, "1");
+  return paramsToUrl(params);
+}
+
+function getDisplayedFacets() {
+  return Array.from(document.querySelectorAll(".facet-info")).map(
+    (el) => el.dataset.column,
+  );
+}
+
+function getColumnClassName(th) {
+  return Array.from(th.classList).find((className) =>
+    className.startsWith("col-"),
+  );
+}
+
+function getColumnCells(th) {
+  var table = th.closest("table");
+  var columnClassName = getColumnClassName(th);
+  if (!table || !columnClassName) {
+    return [];
+  }
+  return Array.from(table.querySelectorAll("td." + columnClassName));
+}
+
+function getColumnMeta(th) {
+  return {
+    columnName: th.dataset.column,
+    columnNotNull: th.dataset.columnNotNull === "1",
+    columnType: th.dataset.columnType,
+    isPk: th.dataset.isPk === "1",
+  };
+}
+
+function getColumnTypeText(th) {
+  var columnType = th.dataset.columnType;
+  if (!columnType) {
+    return null;
+  }
+  var notNull = th.dataset.columnNotNull === "1" ? " NOT NULL" : "";
+  return `Type: ${columnType.toUpperCase()}${notNull}`;
+}
+
+function getSetColumnTypeData() {
+  return window._setColumnTypeData || null;
+}
+
+function getSetColumnTypeConfig(column) {
+  var data = getSetColumnTypeData();
+  if (!data || !data.columns) {
+    return null;
+  }
+  return data.columns[column] || null;
+}
+
+function canSetColumnType() {
+  return !!(getSetColumnTypeData() && window.HTMLDialogElement && window.fetch);
+}
+
+function setColumnTypeActionLabel(column) {
+  var columnConfig = getSetColumnTypeConfig(column);
+  if (!columnConfig) {
+    return null;
+  }
+  return columnConfig.current
+    ? `Custom type: ${columnConfig.current.type}`
+    : "Set custom type";
+}
+
+function createSetColumnTypeOption(value, name, description, checked) {
+  var label = document.createElement("label");
+  label.className = "set-column-type-option";
+
+  var input = document.createElement("input");
+  input.type = "radio";
+  input.name = "set-column-type-choice";
+  input.value = value;
+  input.checked = checked;
+
+  var content = document.createElement("span");
+  content.className = "set-column-type-option-content";
+
+  var title = document.createElement("span");
+  title.className = "set-column-type-option-name";
+  title.textContent = name;
+
+  var detail = document.createElement("span");
+  detail.className = "set-column-type-option-description";
+  detail.textContent = description;
+
+  content.appendChild(title);
+  content.appendChild(detail);
+  label.appendChild(input);
+  label.appendChild(content);
+  return label;
+}
+
+function setSetColumnTypeDialogBusy(state, isBusy) {
+  state.isBusy = isBusy;
+  state.saveButton.disabled = isBusy;
+  state.cancelButton.disabled = isBusy;
+  Array.from(
+    state.optionsWrap.querySelectorAll('input[name="set-column-type-choice"]'),
+  ).forEach(function (input) {
+    input.disabled = isBusy;
+  });
+  state.saveButton.textContent = isBusy ? "Saving..." : "Save";
+}
+
+function clearSetColumnTypeDialogError(state) {
+  state.error.hidden = true;
+  state.error.textContent = "";
+}
+
+function showSetColumnTypeDialogError(state, message) {
+  state.error.hidden = false;
+  state.error.textContent = message;
+}
+
+function ensureSetColumnTypeDialog() {
+  if (setColumnTypeDialogState) {
+    return setColumnTypeDialogState;
+  }
+  if (!window.HTMLDialogElement) {
+    return null;
+  }
+
+  var dialog = document.createElement("dialog");
+  dialog.id = SET_COLUMN_TYPE_DIALOG_ID;
+  dialog.className = "set-column-type-dialog";
+  dialog.setAttribute("aria-labelledby", "set-column-type-title");
+  dialog.innerHTML = `
+    <div class="modal-header">
+      <span class="modal-title" id="set-column-type-title">Set custom type</span>
+      <span class="modal-meta"></span>
+    </div>
+    <p class="set-column-type-status"></p>
+    <p class="set-column-type-error" hidden></p>
+    <div class="set-column-type-options"></div>
+    <div class="modal-footer">
+      <span class="footer-info"></span>
+      <button type="button" class="btn btn-ghost set-column-type-cancel">Cancel</button>
+      <button type="button" class="btn btn-primary set-column-type-save">Save</button>
+    </div>
+  `;
+  document.body.appendChild(dialog);
+
+  setColumnTypeDialogState = {
+    dialog: dialog,
+    meta: dialog.querySelector(".modal-meta"),
+    status: dialog.querySelector(".set-column-type-status"),
+    error: dialog.querySelector(".set-column-type-error"),
+    optionsWrap: dialog.querySelector(".set-column-type-options"),
+    footerInfo: dialog.querySelector(".footer-info"),
+    cancelButton: dialog.querySelector(".set-column-type-cancel"),
+    saveButton: dialog.querySelector(".set-column-type-save"),
+    currentColumn: null,
+    currentConfig: null,
+    isBusy: false,
+  };
+
+  setColumnTypeDialogState.cancelButton.addEventListener("click", function () {
+    if (!setColumnTypeDialogState.isBusy) {
+      dialog.close();
+    }
+  });
+
+  dialog.addEventListener("click", function (ev) {
+    if (ev.target === dialog && !setColumnTypeDialogState.isBusy) {
+      dialog.close();
+    }
+  });
+
+  dialog.addEventListener("cancel", function (ev) {
+    if (setColumnTypeDialogState.isBusy) {
+      ev.preventDefault();
+    }
+  });
+
+  dialog.addEventListener("close", function () {
+    clearSetColumnTypeDialogError(setColumnTypeDialogState);
+    setSetColumnTypeDialogBusy(setColumnTypeDialogState, false);
+  });
+
+  setColumnTypeDialogState.saveButton.addEventListener("click", async function () {
+    var state = setColumnTypeDialogState;
+    var selected = state.dialog.querySelector(
+      'input[name="set-column-type-choice"]:checked',
+    );
+    var selectedType = selected ? selected.value : "";
+    var currentType = state.currentConfig.current
+      ? state.currentConfig.current.type
+      : "";
+
+    if (selectedType === currentType) {
+      state.dialog.close();
+      return;
+    }
+
+    clearSetColumnTypeDialogError(state);
+    setSetColumnTypeDialogBusy(state, true);
+
+    var payload = {
+      column: state.currentColumn,
+      column_type: selectedType ? { type: selectedType } : null,
+    };
+
+    try {
+      var response = await fetch(getSetColumnTypeData().path, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          Accept: "application/json",
+        },
+        body: JSON.stringify(payload),
+      });
+      var data = await response.json();
+      if (!response.ok || data.ok === false) {
+        var message = (data.errors || ["Request failed"]).join(" ");
+        throw new Error(message);
+      }
+      location.reload();
+    } catch (error) {
+      setSetColumnTypeDialogBusy(state, false);
+      showSetColumnTypeDialogError(state, error.message || "Request failed");
+    }
+  });
+
+  return setColumnTypeDialogState;
+}
+
+function openSetColumnTypeDialog(th) {
+  var column = th.dataset.column;
+  var columnConfig = getSetColumnTypeConfig(column);
+  if (!columnConfig) {
+    return;
+  }
+
+  var state = ensureSetColumnTypeDialog();
+  if (!state) {
+    return;
+  }
+
+  clearSetColumnTypeDialogError(state);
+  setSetColumnTypeDialogBusy(state, false);
+  state.currentColumn = column;
+  state.currentConfig = columnConfig;
+  state.status.textContent = `Column: ${column}`;
+  state.meta.textContent = getColumnTypeText(th) || "Type unavailable";
+  state.footerInfo.textContent = columnConfig.current
+    ? `Current custom type: ${columnConfig.current.type}`
+    : "No custom type set.";
+  state.optionsWrap.innerHTML = "";
+
+  var currentType = columnConfig.current ? columnConfig.current.type : "";
+  state.optionsWrap.appendChild(
+    createSetColumnTypeOption(
+      "",
+      "No custom type",
+      "Use standard Datasette rendering without a custom type.",
+      currentType === "",
+    ),
+  );
+
+  columnConfig.options.forEach(function (option) {
+    state.optionsWrap.appendChild(
+      createSetColumnTypeOption(
+        option.name,
+        option.name,
+        option.description,
+        option.name === currentType,
+      ),
+    );
+  });
+
+  if (!columnConfig.options.length) {
+    var emptyState = document.createElement("p");
+    emptyState.className = "set-column-type-empty";
+    emptyState.textContent =
+      "No registered custom types are compatible with this SQLite type.";
+    state.optionsWrap.appendChild(emptyState);
+  }
+
+  if (!state.dialog.open) {
+    state.dialog.showModal();
+  }
+  var selectedOption = state.dialog.querySelector(
+    'input[name="set-column-type-choice"]:checked',
+  );
+  if (selectedOption) {
+    selectedOption.focus();
+  } else {
+    state.saveButton.focus();
+  }
+}
+
+function canChooseColumns() {
+  return !!(
+    document.querySelector("column-chooser") && window._columnChooserData
+  );
+}
+
+function shouldShowShowAllColumns() {
+  var params = getParams();
+  return params.getAll("_nocol").length || params.getAll("_col").length;
+}
+
+function hasMultipleVisibleColumns(manager) {
+  return (
+    Array.from(document.querySelectorAll(manager.selectors.tableHeaders)).filter(
+      (th) => th.dataset.column && th.dataset.isLinkColumn !== "1",
+    ).length > 1
+  );
+}
+
+function buildColumnActionItems(manager, th, options) {
+  options = options || {};
+  var params = getParams();
+  var column = th.dataset.column;
+  var columnActions = [];
+  var isSortable = !!th.querySelector("a");
+  var isFirstColumn = th.parentElement.querySelector("th:first-of-type") === th;
+  var isSinglePk =
+    th.dataset.isPk === "1" &&
+    document.querySelectorAll('th[data-is-pk="1"]').length === 1;
+  var hasBlankValues = getColumnCells(th).some(
+    (el) => el.innerText.trim() === "",
+  );
+
+  if (isSortable && params.get("_sort") !== column) {
+    columnActions.push({
+      label: "Sort ascending",
+      href: sortAscUrl(column),
+    });
+  }
+
+  if (isSortable && params.get("_sort_desc") !== column) {
+    columnActions.push({
+      label: "Sort descending",
+      href: sortDescUrl(column),
+    });
+  }
+
+  if (
+    DATASETTE_ALLOW_FACET &&
+    !isFirstColumn &&
+    !getDisplayedFacets().includes(column) &&
+    !isSinglePk
+  ) {
+    columnActions.push({
+      label: "Facet by this",
+      href: facetUrl(column),
+    });
+  }
+
+  if (options.includeChooseColumns && canChooseColumns()) {
+    columnActions.push({
+      label: "Choose columns",
+      href: "#",
+      onClick:
+        options.onChooseColumns ||
+        function (ev) {
+          ev.preventDefault();
+          openColumnChooser();
+        },
+    });
+  }
+
+  if (canSetColumnType() && getSetColumnTypeConfig(column)) {
+    columnActions.push({
+      label: setColumnTypeActionLabel(column),
+      href: "#",
+      onClick:
+        options.onSetColumnType ||
+        function (ev) {
+          ev.preventDefault();
+          window.setTimeout(function () {
+            openSetColumnTypeDialog(th);
+          }, 0);
+        },
+    });
+  }
+
+  if (th.dataset.isPk !== "1" && hasMultipleVisibleColumns(manager)) {
+    columnActions.push({
+      label: "Hide this column",
+      href: hideColumnUrl(column),
+    });
+  }
+
+  if (options.includeShowAllColumns && shouldShowShowAllColumns()) {
+    columnActions.push({
+      label: "Show all columns",
+      href: showAllColumnsUrl(),
+    });
+  }
+
+  if (params.get(`${column}__notblank`) !== "1" && hasBlankValues) {
+    columnActions.push({
+      label: "Show not-blank rows",
+      href: notBlankUrl(column),
+    });
+  }
+
+  return columnActions.concat(manager.makeColumnActions(getColumnMeta(th)));
+}
+
+function buildColumnActionState(manager, th, options) {
+  return {
+    column: th.dataset.column,
+    columnDescription: th.dataset.columnDescription || null,
+    columnMeta: getColumnMeta(th),
+    columnTypeText: getColumnTypeText(th),
+    actionItems: buildColumnActionItems(manager, th, options),
+  };
+}
+
+function initializeColumnActions(manager) {
+  manager.columnActions = {
+    buildColumnActionState: function (th, options) {
+      return buildColumnActionState(manager, th, options);
+    },
+    buildColumnActionItems: function (th, options) {
+      return buildColumnActionItems(manager, th, options);
+    },
+    canChooseColumns: canChooseColumns,
+    facetUrl: facetUrl,
+    getColumnMeta: getColumnMeta,
+    getColumnTypeText: getColumnTypeText,
+    hideColumnUrl: hideColumnUrl,
+    notBlankUrl: notBlankUrl,
+    shouldShowShowAllColumns: shouldShowShowAllColumns,
+    showAllColumnsUrl: showAllColumnsUrl,
+    sortAscUrl: sortAscUrl,
+    sortDescUrl: sortDescUrl,
+  };
+}
+
+function renderActionLink(itemConfig) {
+  var newLink = document.createElement("a");
+  newLink.textContent = itemConfig.label;
+  newLink.href = itemConfig.href || "#";
+  if (itemConfig.onClick) {
+    newLink.addEventListener("click", itemConfig.onClick);
+  }
+  return newLink;
+}
+
+/** Main initialization function for Datasette Table interactions */
+const initDatasetteTable = function (manager) {
+  // Feature detection
+  if (!window.URLSearchParams) {
+    return;
+  }
+  function closeMenu() {
+    menu.style.display = "none";
+    menu.classList.remove("anim-scale-in");
+  }
+
+  const tableWrapper = document.querySelector(manager.selectors.tableWrapper);
+  if (tableWrapper) {
+    tableWrapper.addEventListener("scroll", closeMenu);
+  }
+  document.body.addEventListener("click", (ev) => {
+    /* was this click outside the menu? */
+    var target = ev.target;
+    while (target && target != menu) {
+      target = target.parentNode;
+    }
+    if (!target) {
+      closeMenu();
+    }
+  });
+
+  function onTableHeaderClick(ev) {
+    ev.preventDefault();
+    ev.stopPropagation();
+    menu.innerHTML = DROPDOWN_HTML;
+    var th = ev.target;
+    while (th.nodeName != "TH") {
+      th = th.parentNode;
+    }
+    var rect = th.getBoundingClientRect();
+    var menuTop = rect.bottom + window.scrollY;
+    var menuLeft = rect.left + window.scrollX;
+    var actionState = manager.columnActions.buildColumnActionState(th, {
+      includeChooseColumns: true,
+      includeShowAllColumns: true,
+      onChooseColumns: function (ev) {
+        ev.preventDefault();
+        closeMenu();
+        openColumnChooser();
+      },
+      onSetColumnType: function (ev) {
+        ev.preventDefault();
+        closeMenu();
+        window.setTimeout(function () {
+          openSetColumnTypeDialog(th);
+        }, 0);
+      },
+    });
+    var menuList = menu.querySelector("ul.dropdown-actions");
+    menuList.innerHTML = "";
+    actionState.actionItems.forEach((itemConfig) => {
+      var menuItem = document.createElement("li");
+      menuItem.appendChild(renderActionLink(itemConfig));
+      menuList.appendChild(menuItem);
+    });
+
+    var columnTypeP = menu.querySelector(".dropdown-column-type");
+    if (actionState.columnTypeText) {
+      columnTypeP.style.display = "block";
+      columnTypeP.innerText = actionState.columnTypeText;
+    } else {
+      columnTypeP.style.display = "none";
+    }
+
+    var columnDescriptionP = menu.querySelector(".dropdown-column-description");
+    if (actionState.columnDescription) {
+      columnDescriptionP.innerText = actionState.columnDescription;
+      columnDescriptionP.style.display = "block";
+    } else {
+      columnDescriptionP.style.display = "none";
+    }
+    menu.style.position = "absolute";
+    menu.style.top = menuTop + 6 + "px";
+    menu.style.left = menuLeft + "px";
+    menu.style.display = "block";
+    menu.classList.add("anim-scale-in");
+
+    // Measure width of menu and adjust position if too far right
+    const menuWidth = menu.offsetWidth;
+    const windowWidth = window.innerWidth;
+    if (menuLeft + menuWidth > windowWidth) {
+      menu.style.left = windowWidth - menuWidth - 20 + "px";
+    }
+    // Align menu .hook arrow with the column cog icon
+    const hook = menu.querySelector(".hook");
+    const icon = th.querySelector(".dropdown-menu-icon");
+    const iconRect = icon.getBoundingClientRect();
+    const hookLeft = iconRect.left - menuLeft + 1 + "px";
+    hook.style.left = hookLeft;
+    // Move the whole menu right if the hook is too far right
+    const menuRect = menu.getBoundingClientRect();
+    if (iconRect.right > menuRect.right) {
+      menu.style.left = iconRect.right - menuWidth + "px";
+      // And move hook tip as well
+      hook.style.left = menuWidth - 13 + "px";
+    }
+  }
+
+  var svg = document.createElement("div");
+  svg.innerHTML = DROPDOWN_ICON_SVG;
+  svg = svg.querySelector("*");
+  svg.classList.add("dropdown-menu-icon");
+  var menu = document.createElement("div");
+  menu.innerHTML = DROPDOWN_HTML;
+  menu = menu.querySelector("*");
+  menu.style.position = "absolute";
+  menu.style.display = "none";
+  document.body.appendChild(menu);
+
+  var ths = Array.from(
+    document.querySelectorAll(manager.selectors.tableHeaders),
+  );
+  ths.forEach((th) => {
+    if (!th.querySelector("a")) {
+      return;
+    }
+    var icon = svg.cloneNode(true);
+    icon.addEventListener("click", onTableHeaderClick);
+    th.appendChild(icon);
+  });
+};
+
+/* Add x buttons to the filter rows */
+function addButtonsToFilterRows(manager) {
+  var x = "✖";
+  var rows = Array.from(
+    document.querySelectorAll(manager.selectors.filterRow),
+  ).filter((el) => el.querySelector(".filter-op"));
+  rows.forEach((row) => {
+    var a = document.createElement("a");
+    a.setAttribute("href", "#");
+    a.setAttribute("aria-label", "Remove this filter");
+    a.style.textDecoration = "none";
+    a.innerText = x;
+    a.addEventListener("click", (ev) => {
+      ev.preventDefault();
+      let row = ev.target.closest("div");
+      row.querySelector("select").value = "";
+      row.querySelector(".filter-op select").value = "exact";
+      row.querySelector("input.filter-value").value = "";
+      ev.target.closest("a").style.display = "none";
+    });
+    row.appendChild(a);
+    var column = row.querySelector("select");
+    if (!column.value) {
+      a.style.display = "none";
+    }
+  });
+}
+
+/* Set up datalist autocomplete for filter values */
+function initAutocompleteForFilterValues(manager) {
+  function createDataLists() {
+    var facetResults = document.querySelectorAll(
+      manager.selectors.facetResults,
+    );
+    Array.from(facetResults).forEach(function (facetResult) {
+      // Use link text from all links in the facet result
+      var links = Array.from(
+        facetResult.querySelectorAll("li:not(.facet-truncated) a"),
+      );
+      // Create a datalist element
+      var datalist = document.createElement("datalist");
+      datalist.id = "datalist-" + facetResult.dataset.column;
+      // Create an option element for each link text
+      links.forEach(function (link) {
+        var option = document.createElement("option");
+        option.label = link.innerText;
+        option.value = link.dataset.facetValue;
+        datalist.appendChild(option);
+      });
+      // Add the datalist to the facet result
+      facetResult.appendChild(datalist);
+    });
+  }
+  createDataLists();
+  // When any select with name=_filter_column changes, update the datalist
+  document.body.addEventListener("change", function (event) {
+    if (event.target.name === "_filter_column") {
+      event.target
+        .closest(manager.selectors.filterRow)
+        .querySelector(".filter-value")
+        .setAttribute("list", "datalist-" + event.target.value);
+    }
+  });
+}
+
+/** Open the column-chooser web component */
+function openColumnChooser() {
+  var chooser = document.querySelector("column-chooser");
+  var data = window._columnChooserData;
+  if (!chooser || !data) return;
+
+  var nonPkColumns = data.allColumns.filter(function (col) {
+    return data.primaryKeys.indexOf(col) === -1;
+  });
+  var selected = data.selectedColumns.filter(function (col) {
+    return data.primaryKeys.indexOf(col) === -1;
+  });
+
+  chooser.open({
+    columns: nonPkColumns,
+    selected: selected,
+    onApply: function (cols) {
+      var params = new URLSearchParams(location.search);
+      params.delete("_col");
+      params.delete("_nocol");
+      params.delete("_next");
+
+      if (cols.length === nonPkColumns.length) {
+        // Check if order matches original - if so, no params needed
+        var orderMatches = cols.every(function (col, i) {
+          return col === nonPkColumns[i];
+        });
+        if (!orderMatches) {
+          cols.forEach(function (col) {
+            params.append("_col", col);
+          });
+        }
+      } else {
+        cols.forEach(function (col) {
+          params.append("_col", col);
+        });
+      }
+      var qs = params.toString();
+      location.href = qs ? "?" + qs : location.pathname;
+    },
+  });
+}
+
+// Ensures Table UI is initialized only after the Manager is ready.
+document.addEventListener("datasette_init", function (evt) {
+  const { detail: manager } = evt;
+
+  initializeColumnActions(manager);
+
+  // Main table
+  initDatasetteTable(manager);
+
+  // Other UI functions with interactive JS needs
+  addButtonsToFilterRows(manager);
+  initAutocompleteForFilterValues(manager);
+});

datasette/stored_queries.py

@@ -0,0 +1,581 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+import json
+from typing import Any, Iterable
+
+from .utils import tilde_encode, urlsafe_components
+
+UNCHANGED = object()
+
+
+QUERY_OPTION_FIELDS = (
+    "hide_sql",
+    "fragment",
+    "on_success_message",
+    "on_success_message_sql",
+    "on_success_redirect",
+    "on_error_message",
+    "on_error_redirect",
+)
+
+
+@dataclass
+class StoredQuery:
+    database: str
+    name: str
+    sql: str
+    title: str | None
+    description: str | None
+    description_html: str | None
+    hide_sql: bool
+    fragment: str | None
+    parameters: list[str]
+    is_write: bool
+    is_private: bool
+    is_trusted: bool
+    source: str
+    owner_id: str | None
+    on_success_message: str | None
+    on_success_message_sql: str | None
+    on_success_redirect: str | None
+    on_error_message: str | None
+    on_error_redirect: str | None
+    private: bool | None = None
+
+
+@dataclass
+class StoredQueryPage:
+    queries: list[StoredQuery]
+    next: str | None
+    has_more: bool
+    limit: int
+
+
+def stored_query_to_dict(query: StoredQuery) -> dict[str, Any]:
+    data = {
+        "database": query.database,
+        "name": query.name,
+        "sql": query.sql,
+        "title": query.title,
+        "description": query.description,
+        "description_html": query.description_html,
+        "hide_sql": query.hide_sql,
+        "fragment": query.fragment,
+        "params": list(query.parameters),
+        "parameters": list(query.parameters),
+        "is_write": query.is_write,
+        "is_private": query.is_private,
+        "is_trusted": query.is_trusted,
+        "source": query.source,
+        "owner_id": query.owner_id,
+        "on_success_message": query.on_success_message,
+        "on_success_message_sql": query.on_success_message_sql,
+        "on_success_redirect": query.on_success_redirect,
+        "on_error_message": query.on_error_message,
+        "on_error_redirect": query.on_error_redirect,
+    }
+    if query.private is not None:
+        data["private"] = query.private
+    return data
+
+
+def stored_query_page_to_dict(page: StoredQueryPage) -> dict[str, Any]:
+    return {
+        "queries": [stored_query_to_dict(query) for query in page.queries],
+        "next": page.next,
+        "has_more": page.has_more,
+        "limit": page.limit,
+    }
+
+
+async def save_queries_from_config(datasette: Any) -> None:
+    # Apply configured query entries from datasette.yaml to the internal table.
+    await datasette.get_internal_database().execute_write(
+        "DELETE FROM queries WHERE source = 'config'"
+    )
+    for dbname, db_config in ((datasette.config or {}).get("databases") or {}).items():
+        for query_name, query_config in (db_config.get("queries") or {}).items():
+            if not isinstance(query_config, dict):
+                query_config = {"sql": query_config}
+            await datasette.add_query(
+                dbname,
+                query_name,
+                query_config["sql"],
+                title=query_config.get("title"),
+                description=query_config.get("description"),
+                description_html=query_config.get("description_html"),
+                hide_sql=bool(query_config.get("hide_sql")),
+                fragment=query_config.get("fragment"),
+                parameters=query_config.get("params"),
+                is_write=bool(query_config.get("write")),
+                is_trusted=bool(query_config.get("is_trusted", True)),
+                source="config",
+                on_success_message=query_config.get("on_success_message"),
+                on_success_message_sql=query_config.get("on_success_message_sql"),
+                on_success_redirect=query_config.get("on_success_redirect"),
+                on_error_message=query_config.get("on_error_message"),
+                on_error_redirect=query_config.get("on_error_redirect"),
+            )
+
+
+def query_row_to_stored_query(
+    row: Any, private: bool | None = None
+) -> StoredQuery | None:
+    if row is None:
+        return None
+    parameters = json.loads(row["parameters"] or "[]")
+    options = json.loads(row["options"] or "{}")
+    return StoredQuery(
+        database=row["database_name"],
+        name=row["name"],
+        sql=row["sql"],
+        title=row["title"],
+        description=row["description"],
+        description_html=row["description_html"],
+        hide_sql=bool(options.get("hide_sql")),
+        fragment=options.get("fragment"),
+        parameters=parameters,
+        is_write=bool(row["is_write"]),
+        is_private=bool(row["is_private"]),
+        is_trusted=bool(row["is_trusted"]),
+        source=row["source"],
+        owner_id=row["owner_id"],
+        on_success_message=options.get("on_success_message"),
+        on_success_message_sql=options.get("on_success_message_sql"),
+        on_success_redirect=options.get("on_success_redirect"),
+        on_error_message=options.get("on_error_message"),
+        on_error_redirect=options.get("on_error_redirect"),
+        private=private,
+    )
+
+
+def query_options_json(options: dict[str, Any]) -> str:
+    options_dict = {}
+    for field in QUERY_OPTION_FIELDS:
+        value = options.get(field)
+        if field == "hide_sql":
+            if value:
+                options_dict[field] = True
+        elif value is not None:
+            options_dict[field] = value
+    return json.dumps(options_dict, sort_keys=True)
+
+
+async def add_query(
+    datasette: Any,
+    database: str,
+    name: str,
+    sql: str,
+    *,
+    title: str | None = None,
+    description: str | None = None,
+    description_html: str | None = None,
+    hide_sql: bool = False,
+    fragment: str | None = None,
+    parameters: Iterable[str] | None = None,
+    is_write: bool = False,
+    is_private: bool = False,
+    is_trusted: bool = False,
+    source: str = "plugin",
+    owner_id: str | None = None,
+    on_success_message: str | None = None,
+    on_success_message_sql: str | None = None,
+    on_success_redirect: str | None = None,
+    on_error_message: str | None = None,
+    on_error_redirect: str | None = None,
+    replace: bool = True,
+) -> None:
+    parameters_json = json.dumps(list(parameters or []))
+    options_json = query_options_json(
+        {
+            "hide_sql": hide_sql,
+            "fragment": fragment,
+            "on_success_message": on_success_message,
+            "on_success_message_sql": on_success_message_sql,
+            "on_success_redirect": on_success_redirect,
+            "on_error_message": on_error_message,
+            "on_error_redirect": on_error_redirect,
+        }
+    )
+    sql_statement = """
+        INSERT INTO queries (
+            database_name, name, sql, title, description, description_html,
+            options, parameters, is_write, is_private, is_trusted, source, owner_id
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    """
+    if replace:
+        sql_statement += """
+            ON CONFLICT(database_name, name) DO UPDATE SET
+                sql = excluded.sql,
+                title = excluded.title,
+                description = excluded.description,
+                description_html = excluded.description_html,
+                options = excluded.options,
+                parameters = excluded.parameters,
+                is_write = excluded.is_write,
+                is_private = excluded.is_private,
+                is_trusted = excluded.is_trusted,
+                source = excluded.source,
+                owner_id = excluded.owner_id,
+                updated_at = CURRENT_TIMESTAMP
+        """
+    await datasette.get_internal_database().execute_write(
+        sql_statement,
+        [
+            database,
+            name,
+            sql,
+            title,
+            description,
+            description_html,
+            options_json,
+            parameters_json,
+            int(bool(is_write)),
+            int(bool(is_private)),
+            int(bool(is_trusted)),
+            source,
+            owner_id,
+        ],
+    )
+
+
+async def update_query(
+    datasette: Any,
+    database: str,
+    name: str,
+    *,
+    sql=UNCHANGED,
+    title=UNCHANGED,
+    description=UNCHANGED,
+    description_html=UNCHANGED,
+    hide_sql=UNCHANGED,
+    fragment=UNCHANGED,
+    parameters=UNCHANGED,
+    is_write=UNCHANGED,
+    is_private=UNCHANGED,
+    is_trusted=UNCHANGED,
+    source=UNCHANGED,
+    owner_id=UNCHANGED,
+    on_success_message=UNCHANGED,
+    on_success_message_sql=UNCHANGED,
+    on_success_redirect=UNCHANGED,
+    on_error_message=UNCHANGED,
+    on_error_redirect=UNCHANGED,
+) -> None:
+    fields = {
+        "sql": sql,
+        "title": title,
+        "description": description,
+        "description_html": description_html,
+        "parameters": parameters,
+        "is_write": is_write,
+        "is_private": is_private,
+        "is_trusted": is_trusted,
+        "source": source,
+        "owner_id": owner_id,
+    }
+    option_fields = {
+        "hide_sql": hide_sql,
+        "fragment": fragment,
+        "on_success_message": on_success_message,
+        "on_success_message_sql": on_success_message_sql,
+        "on_success_redirect": on_success_redirect,
+        "on_error_message": on_error_message,
+        "on_error_redirect": on_error_redirect,
+    }
+    updates = []
+    params = []
+    for field, value in fields.items():
+        if value is UNCHANGED:
+            continue
+        if field in {"is_write", "is_private", "is_trusted"}:
+            value = int(bool(value))
+        elif field == "parameters":
+            value = json.dumps(list(value or []))
+        updates.append(f"{field} = ?")
+        params.append(value)
+    changed_options = {
+        field: value for field, value in option_fields.items() if value is not UNCHANGED
+    }
+    if changed_options:
+        rows = await datasette.get_internal_database().execute(
+            """
+            SELECT options FROM queries
+            WHERE database_name = ? AND name = ?
+            """,
+            [database, name],
+        )
+        row = rows.first()
+        options = json.loads(row["options"] or "{}") if row is not None else {}
+        for field, value in changed_options.items():
+            if field == "hide_sql":
+                if value:
+                    options[field] = True
+                else:
+                    options.pop(field, None)
+            elif value is None:
+                options.pop(field, None)
+            else:
+                options[field] = value
+        updates.append("options = ?")
+        params.append(json.dumps(options, sort_keys=True))
+    if not updates:
+        return
+    updates.append("updated_at = CURRENT_TIMESTAMP")
+    params.extend([database, name])
+    await datasette.get_internal_database().execute_write(
+        """
+        UPDATE queries
+        SET {}
+        WHERE database_name = ? AND name = ?
+        """.format(", ".join(updates)),
+        params,
+    )
+
+
+async def remove_query(
+    datasette: Any, database: str, name: str, source: str | None = None
+) -> None:
+    sql = "DELETE FROM queries WHERE database_name = ? AND name = ?"
+    params = [database, name]
+    if source is not None:
+        sql += " AND source = ?"
+        params.append(source)
+    await datasette.get_internal_database().execute_write(sql, params)
+
+
+async def get_query(datasette: Any, database: str, name: str) -> StoredQuery | None:
+    rows = await datasette.get_internal_database().execute(
+        """
+        SELECT * FROM queries
+        WHERE database_name = ? AND name = ?
+        """,
+        [database, name],
+    )
+    return query_row_to_stored_query(rows.first())
+
+
+async def count_queries(
+    datasette: Any,
+    database: str | None = None,
+    *,
+    actor: dict[str, Any] | None = None,
+    q: str | None = None,
+    is_write: bool | None = None,
+    is_private: bool | None = None,
+    is_trusted: bool | None = None,
+    source: str | None = None,
+    owner_id: str | None = None,
+) -> int:
+    allowed_sql, allowed_params = await datasette.allowed_resources_sql(
+        action="view-query",
+        actor=actor,
+        parent=database,
+    )
+    params = dict(allowed_params)
+    where_clauses = []
+    if database is not None:
+        params["query_database"] = database
+        where_clauses.append("q.database_name = :query_database")
+
+    if q:
+        where_clauses.append("""
+            (
+                q.name LIKE :query_search
+                OR q.title LIKE :query_search
+                OR q.description LIKE :query_search
+                OR q.sql LIKE :query_search
+            )
+            """)
+        params["query_search"] = "%{}%".format(q)
+    if is_write is not None:
+        where_clauses.append("q.is_write = :query_is_write")
+        params["query_is_write"] = int(bool(is_write))
+    if is_private is not None:
+        where_clauses.append("q.is_private = :query_is_private")
+        params["query_is_private"] = int(bool(is_private))
+    if is_trusted is not None:
+        where_clauses.append("q.is_trusted = :query_is_trusted")
+        params["query_is_trusted"] = int(bool(is_trusted))
+    if source is not None:
+        where_clauses.append("q.source = :query_source")
+        params["query_source"] = source
+    if owner_id is not None:
+        where_clauses.append("q.owner_id = :query_owner_id")
+        params["query_owner_id"] = owner_id
+
+    row = (
+        await datasette.get_internal_database().execute(
+            """
+            SELECT count(*) AS count
+            FROM queries q
+            JOIN (
+                {allowed_sql}
+            ) allowed
+              ON allowed.parent = q.database_name
+             AND allowed.child = q.name
+            WHERE {where}
+            """.format(
+                allowed_sql=allowed_sql,
+                where=" AND ".join(where_clauses) or "1 = 1",
+            ),
+            params,
+        )
+    ).first()
+    return row["count"]
+
+
+async def list_queries(
+    datasette: Any,
+    database: str | None = None,
+    *,
+    actor: dict[str, Any] | None = None,
+    limit: int = 50,
+    cursor: str | None = None,
+    q: str | None = None,
+    is_write: bool | None = None,
+    is_private: bool | None = None,
+    is_trusted: bool | None = None,
+    source: str | None = None,
+    owner_id: str | None = None,
+    include_private: bool = False,
+) -> StoredQueryPage:
+    limit = min(max(1, int(limit)), 1000)
+    allowed_sql, allowed_params = await datasette.allowed_resources_sql(
+        action="view-query",
+        actor=actor,
+        parent=database,
+        include_is_private=include_private,
+    )
+    params = dict(allowed_params)
+    params.update({"limit": limit + 1})
+    sort_key_sql = "lower(coalesce(nullif(q.title, ''), q.name))"
+    where_clauses = []
+    order_by = "q.database_name, sort_key, q.name"
+    if database is not None:
+        params["query_database"] = database
+        where_clauses.append("q.database_name = :query_database")
+        order_by = "sort_key, q.name"
+
+    if cursor:
+        try:
+            components = urlsafe_components(cursor)
+        except ValueError:
+            components = []
+        if database is None and len(components) == 3:
+            where_clauses.append("""
+                (
+                    q.database_name > :cursor_database
+                    OR (
+                        q.database_name = :cursor_database
+                        AND (
+                            {sort_key_sql} > :cursor_sort_key
+                            OR (
+                                {sort_key_sql} = :cursor_sort_key
+                                AND q.name > :cursor_name
+                            )
+                        )
+                    )
+                )
+                """.format(sort_key_sql=sort_key_sql))
+            params["cursor_database"] = components[0]
+            params["cursor_sort_key"] = components[1]
+            params["cursor_name"] = components[2]
+        elif database is not None and len(components) == 2:
+            where_clauses.append("""
+                (
+                    {sort_key_sql} > :cursor_sort_key
+                    OR (
+                        {sort_key_sql} = :cursor_sort_key
+                        AND q.name > :cursor_name
+                    )
+                )
+                """.format(sort_key_sql=sort_key_sql))
+            params["cursor_sort_key"] = components[0]
+            params["cursor_name"] = components[1]
+
+    if q:
+        where_clauses.append("""
+            (
+                q.name LIKE :query_search
+                OR q.title LIKE :query_search
+                OR q.description LIKE :query_search
+                OR q.sql LIKE :query_search
+            )
+            """)
+        params["query_search"] = "%{}%".format(q)
+    if is_write is not None:
+        where_clauses.append("q.is_write = :query_is_write")
+        params["query_is_write"] = int(bool(is_write))
+    if is_private is not None:
+        where_clauses.append("q.is_private = :query_is_private")
+        params["query_is_private"] = int(bool(is_private))
+    if is_trusted is not None:
+        where_clauses.append("q.is_trusted = :query_is_trusted")
+        params["query_is_trusted"] = int(bool(is_trusted))
+    if source is not None:
+        where_clauses.append("q.source = :query_source")
+        params["query_source"] = source
+    if owner_id is not None:
+        where_clauses.append("q.owner_id = :query_owner_id")
+        params["query_owner_id"] = owner_id
+
+    private_select = ", allowed.is_private AS private" if include_private else ""
+    rows = list(
+        (
+            await datasette.get_internal_database().execute(
+                """
+                SELECT q.*, {sort_key_sql} AS sort_key{private_select}
+                FROM queries q
+                JOIN (
+                    {allowed_sql}
+                ) allowed
+                  ON allowed.parent = q.database_name
+                 AND allowed.child = q.name
+                WHERE {where}
+                ORDER BY {order_by}
+                LIMIT :limit
+                """.format(
+                    allowed_sql=allowed_sql,
+                    private_select=private_select,
+                    sort_key_sql=sort_key_sql,
+                    where=" AND ".join(where_clauses) or "1 = 1",
+                    order_by=order_by,
+                ),
+                params,
+            )
+        ).rows
+    )
+    has_more = len(rows) > limit
+    if has_more:
+        rows = rows[:limit]
+
+    queries = []
+    for row in rows:
+        query = query_row_to_stored_query(
+            row, private=bool(row["private"]) if include_private else None
+        )
+        assert query is not None
+        queries.append(query)
+
+    next_token = None
+    if has_more and rows:
+        last_row = rows[-1]
+        if database is None:
+            next_token = "{},{},{}".format(
+                tilde_encode(last_row["database_name"]),
+                tilde_encode(last_row["sort_key"]),
+                tilde_encode(last_row["name"]),
+            )
+        else:
+            next_token = "{},{}".format(
+                tilde_encode(last_row["sort_key"]),
+                tilde_encode(last_row["name"]),
+            )
+    return StoredQueryPage(
+        queries=queries,
+        next=next_token,
+        has_more=has_more,
+        limit=limit,
+    )

datasette/templates/_action_menu.html

@@ -0,0 +1,28 @@
+{% if action_links %}
+<div class="page-action-menu">
+<details class="actions-menu-links details-menu">
+    <summary aria-haspopup="menu" aria-expanded="false">
+        <div class="icon-text">
+            <svg class="icon" aria-labelledby="actions-menu-links-title" role="img" style="color: #fff" xmlns="http://www.w3.org/2000/svg" width="28" height="28" viewBox="0 0 28 28" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
+                <title id="actions-menu-links-title">{{ action_title }}</title>
+                <circle cx="12" cy="12" r="3"></circle>
+                <path d="M19.4 15a1.65 1.65 0 0 0 .33 1.82l.06.06a2 2 0 0 1 0 2.83 2 2 0 0 1-2.83 0l-.06-.06a1.65 1.65 0 0 0-1.82-.33 1.65 1.65 0 0 0-1 1.51V21a2 2 0 0 1-2 2 2 2 0 0 1-2-2v-.09A1.65 1.65 0 0 0 9 19.4a1.65 1.65 0 0 0-1.82.33l-.06.06a2 2 0 0 1-2.83 0 2 2 0 0 1 0-2.83l.06-.06a1.65 1.65 0 0 0 .33-1.82 1.65 1.65 0 0 0-1.51-1H3a2 2 0 0 1-2-2 2 2 0 0 1 2-2h.09A1.65 1.65 0 0 0 4.6 9a1.65 1.65 0 0 0-.33-1.82l-.06-.06a2 2 0 0 1 0-2.83 2 2 0 0 1 2.83 0l.06.06a1.65 1.65 0 0 0 1.82.33H9a1.65 1.65 0 0 0 1-1.51V3a2 2 0 0 1 2-2 2 2 0 0 1 2 2v.09a1.65 1.65 0 0 0 1 1.51 1.65 1.65 0 0 0 1.82-.33l.06-.06a2 2 0 0 1 2.83 0 2 2 0 0 1 0 2.83l-.06.06a1.65 1.65 0 0 0-.33 1.82V9a1.65 1.65 0 0 0 1.51 1H21a2 2 0 0 1 2 2 2 2 0 0 1-2 2h-.09a1.65 1.65 0 0 0-1.51 1z"></path>
+            </svg>
+            <span>{{ action_title }}</span>
+        </div>
+    </summary>
+    <div class="dropdown-menu">
+        <div class="hook"></div>
+        <ul role="menu">
+            {% for link in action_links %}
+            <li role="none"><a href="{{ link.href }}" role="menuitem" tabindex="-1">{{ link.label }}
+            {% if link.description %}
+                <p class="dropdown-description">{{ link.description }}</p>
+            {% endif %}</a>
+            </li>
+            {% endfor %}
+        </ul>
+    </div>
+</details>
+</div>
+{% endif %}

datasette/templates/_close_open_menus.html

@@ -0,0 +1,62 @@
+<script>
+document.body.addEventListener('click', (ev) => {
+    /* Close any open details elements that this click is outside of */
+    var target = ev.target;
+    var detailsClickedWithin = null;
+    while (target && target.tagName != 'DETAILS') {
+        target = target.parentNode;
+    }
+    if (target && target.tagName == 'DETAILS') {
+        detailsClickedWithin = target;
+    }
+    Array.from(document.querySelectorAll('details.details-menu')).filter(
+        (details) => details.open && details != detailsClickedWithin
+    ).forEach(details => details.open = false);
+});
+
+/* Sync aria-expanded and add keyboard navigation for details-menu elements */
+document.querySelectorAll('details.details-menu').forEach(function(details) {
+    var summary = details.querySelector('summary');
+    details.addEventListener('toggle', function() {
+        if (summary) {
+            summary.setAttribute('aria-expanded', details.open ? 'true' : 'false');
+        }
+        if (details.open) {
+            /* Focus first menu item when menu opens */
+            var firstItem = details.querySelector('[role="menuitem"]');
+            if (firstItem) { firstItem.focus(); }
+        }
+    });
+});
+
+document.body.addEventListener('keydown', function(ev) {
+    /* Keyboard navigation for open details-menu elements */
+    var openDetails = Array.from(document.querySelectorAll('details.details-menu[open]'));
+    if (!openDetails.length) { return; }
+
+    if (ev.key === 'Escape') {
+        openDetails.forEach(function(details) {
+            details.open = false;
+            var summary = details.querySelector('summary');
+            if (summary) { summary.focus(); }
+        });
+        return;
+    }
+
+    if (ev.key === 'ArrowDown' || ev.key === 'ArrowUp') {
+        var focused = document.activeElement;
+        openDetails.forEach(function(details) {
+            var items = Array.from(details.querySelectorAll('[role="menuitem"]'));
+            if (!items.length) { return; }
+            var idx = items.indexOf(focused);
+            if (idx === -1) { return; }
+            ev.preventDefault();
+            if (ev.key === 'ArrowDown') {
+                items[(idx + 1) % items.length].focus();
+            } else {
+                items[(idx - 1 + items.length) % items.length].focus();
+            }
+        });
+    }
+});
+</script>

datasette/templates/_codemirror.html

@@ -1,8 +1,16 @@
 <script src="{{ base_url }}-/static/sql-formatter-2.3.3.min.js" defer></script>
-<script src="{{ base_url }}-/static/codemirror-5.31.0.js"></script>
-<link rel="stylesheet" href="{{ base_url }}-/static/codemirror-5.31.0-min.css" />
-<script src="{{ base_url }}-/static/codemirror-5.31.0-sql.min.js"></script>
+<script src="{{ base_url }}-/static/cm-editor-6.0.1.bundle.js"></script>
 <style>
-    .CodeMirror { height: auto; min-height: 70px; width: 80%; border: 1px solid #ddd; }
-    .CodeMirror-scroll { max-height: 200px; }
+  .cm-editor {
+    resize: both;
+    overflow: hidden;
+    width: 80%;
+    border: 1px solid #ddd;
+  }
+  /* Fix autocomplete icon positioning. The icon element gets border-box sizing set due to
+     the global reset, but this causes overlapping icon and text. Markup:
+     `<div class="cm-completionIcon cm-completionIcon-keyword" aria-hidden="true"></div>` */
+  .cm-completionIcon {
+    box-sizing: content-box;
+  }
 </style>

datasette/templates/_codemirror_foot.html

@@ -1,37 +1,42 @@
 <script>
-window.onload = () => {
+  {% if table_columns %}
+  const schema = {{ table_columns|tojson(2) }};
+  {% else %}
+  const schema = {};
+  {% endif %}
+
+  window.addEventListener("DOMContentLoaded", () => {
     const sqlFormat = document.querySelector("button#sql-format");
     const readOnly = document.querySelector("pre#sql-query");
     const sqlInput = document.querySelector("textarea#sql-editor");
     if (sqlFormat && !readOnly) {
-        sqlFormat.hidden = false;
+      sqlFormat.hidden = false;
     }
     if (sqlInput) {
-        var editor = CodeMirror.fromTextArea(sqlInput, {
-          lineNumbers: true,
-          mode: "text/x-sql",
-          lineWrapping: true,
+      var editor = (window.editor = cm.editorFromTextArea(sqlInput, {
+        schema,
+      }));
+      if (sqlFormat) {
+        sqlFormat.addEventListener("click", (ev) => {
+          const formatted = sqlFormatter.format(editor.state.doc.toString());
+          editor.dispatch({
+            changes: {
+              from: 0,
+              to: editor.state.doc.length,
+              insert: formatted,
+            },
+          });
         });
-        editor.setOption("extraKeys", {
-          "Shift-Enter": function() {
-            document.getElementsByClassName("sql")[0].submit();
-          },
-          Tab: false
-        });
-        if (sqlFormat) {
-            sqlFormat.addEventListener("click", ev => {
-                editor.setValue(sqlFormatter.format(editor.getValue()));
-            })
-        }
+      }
     }
     if (sqlFormat && readOnly) {
-        const formatted = sqlFormatter.format(readOnly.innerHTML);
-        if (formatted != readOnly.innerHTML) {
-            sqlFormat.hidden = false;
-            sqlFormat.addEventListener("click", ev => {
-                readOnly.innerHTML = formatted;
-            })
-        }
+      const formatted = sqlFormatter.format(readOnly.innerHTML);
+      if (formatted != readOnly.innerHTML) {
+        sqlFormat.hidden = false;
+        sqlFormat.addEventListener("click", (ev) => {
+          readOnly.innerHTML = formatted;
+        });
+      }
     }
-}
+  });
 </script>

datasette/templates/_crumbs.html

@@ -0,0 +1,15 @@
+{% macro nav(request, database=None, table=None) -%}
+{% if crumb_items is defined %}
+  {% set items=crumb_items(request=request, database=database, table=table) %}
+  {% if items %}
+    <p class="crumbs">
+      {% for item in items %}
+        <a href="{{ item.href }}">{{ item.label }}</a>
+        {% if not loop.last %}
+          /
+        {% endif %}
+      {% endfor %}
+    </p>
+  {% endif %}
+{% endif %}
+{%- endmacro %}

datasette/templates/_debug_common_functions.html

@@ -0,0 +1,50 @@
+<script>
+// Common utility functions for debug pages
+
+// Populate form from URL parameters on page load
+function populateFormFromURL() {
+    const params = new URLSearchParams(window.location.search);
+
+    const action = params.get('action');
+    if (action) {
+        const actionField = document.getElementById('action');
+        if (actionField) {
+            actionField.value = action;
+        }
+    }
+
+    const parent = params.get('parent');
+    if (parent) {
+        const parentField = document.getElementById('parent');
+        if (parentField) {
+            parentField.value = parent;
+        }
+    }
+
+    const child = params.get('child');
+    if (child) {
+        const childField = document.getElementById('child');
+        if (childField) {
+            childField.value = child;
+        }
+    }
+
+    const pageSize = params.get('page_size');
+    if (pageSize) {
+        const pageSizeField = document.getElementById('page_size');
+        if (pageSizeField) {
+            pageSizeField.value = pageSize;
+        }
+    }
+
+    return params;
+}
+
+// HTML escape function
+function escapeHtml(text) {
+    if (text === null || text === undefined) return '';
+    const div = document.createElement('div');
+    div.textContent = text;
+    return div.innerHTML;
+}
+</script>

datasette/templates/_description_source_license.html

@@ -1,6 +1,6 @@
-{% if metadata.description_html or metadata.description %}
+{% if metadata.get("description_html") or metadata.get("description") %}
     <div class="metadata-description">
-        {% if metadata.description_html %}
+        {% if metadata.get("description_html") %}
             {{ metadata.description_html|safe }}
         {% else %}
             {{ metadata.description }}

datasette/templates/_execute_write_analysis_scripts.html

@@ -0,0 +1,111 @@
+<script>
+window.datasetteSqlAnalysis = (() => {
+  if (
+    window.datasetteSqlAnalysis &&
+    window.datasetteSqlAnalysis.renderAnalysis
+  ) {
+    return window.datasetteSqlAnalysis;
+  }
+
+  function appendCodeCell(row, value, emptyText) {
+    const cell = document.createElement("td");
+    if (value) {
+      const code = document.createElement("code");
+      code.textContent = value;
+      cell.appendChild(code);
+    } else if (emptyText) {
+      appendNotApplicable(cell);
+    }
+    row.appendChild(cell);
+  }
+
+  function appendNotApplicable(cell) {
+    const notApplicable = document.createElement("span");
+    notApplicable.className = "execute-write-analysis-na";
+    notApplicable.textContent = "n/a";
+    cell.appendChild(notApplicable);
+  }
+
+  function renderAnalysis(section, data) {
+    if (!section) {
+      return;
+    }
+    section.replaceChildren();
+    if (data.has_sql === false) {
+      section.hidden = true;
+      return;
+    }
+    section.hidden = false;
+
+    const heading = document.createElement("h2");
+    heading.textContent = "Query operations";
+    section.appendChild(heading);
+
+    if (data.analysis_error) {
+      const error = document.createElement("p");
+      error.className = "message-error";
+      error.textContent = data.analysis_error;
+      section.appendChild(error);
+      return;
+    }
+
+    const rows = data.analysis_rows || [];
+    if (!rows.length) {
+      const empty = document.createElement("p");
+      empty.textContent =
+        "Analysis will show each affected table and required permission.";
+      section.appendChild(empty);
+      return;
+    }
+
+    const wrapper = document.createElement("div");
+    wrapper.className = "table-wrapper";
+    const table = document.createElement("table");
+    table.className = "execute-write-analysis";
+    const thead = document.createElement("thead");
+    const headerRow = document.createElement("tr");
+    [
+      "Operation",
+      "Database",
+      "Table",
+      "Required permission",
+      "Allowed",
+    ].forEach((label) => {
+      const th = document.createElement("th");
+      th.scope = "col";
+      th.textContent = label;
+      headerRow.appendChild(th);
+    });
+    thead.appendChild(headerRow);
+    table.appendChild(thead);
+
+    const tbody = document.createElement("tbody");
+    rows.forEach((analysisRow) => {
+      const row = document.createElement("tr");
+      appendCodeCell(row, analysisRow.operation);
+      appendCodeCell(row, analysisRow.database);
+      appendCodeCell(row, analysisRow.table);
+      appendCodeCell(row, analysisRow.required_permission, "n/a");
+
+      const allowedCell = document.createElement("td");
+      if (analysisRow.allowed !== null && analysisRow.allowed !== undefined) {
+        const allowed = document.createElement("span");
+        allowed.className = analysisRow.allowed
+          ? "execute-write-analysis-allowed"
+          : "execute-write-analysis-denied";
+        allowed.textContent = analysisRow.allowed ? "yes" : "no";
+        allowedCell.appendChild(allowed);
+      } else {
+        appendNotApplicable(allowedCell);
+      }
+      row.appendChild(allowedCell);
+      tbody.appendChild(row);
+    });
+    table.appendChild(tbody);
+    wrapper.appendChild(table);
+    section.appendChild(wrapper);
+  }
+
+  return { renderAnalysis };
+})();
+</script>

datasette/templates/_execute_write_analysis_styles.html

@@ -0,0 +1,41 @@
+<style>
+.execute-write-analysis {
+    border-collapse: collapse;
+    font-size: 0.9rem;
+    margin: 0.25rem 0 1rem;
+    min-width: 44rem;
+}
+.execute-write-analysis th,
+.execute-write-analysis td {
+    border-bottom: 1px solid #d7dde5;
+    padding: 0.45rem 0.7rem;
+    text-align: left;
+    vertical-align: top;
+}
+.execute-write-analysis th {
+    background-color: #edf6fb;
+    border-top: 1px solid #d7dde5;
+    color: #39445a;
+    font-weight: 700;
+}
+.execute-write-analysis tbody tr:nth-child(even) {
+    background-color: rgba(39, 104, 144, 0.05);
+}
+.execute-write-analysis code {
+    background: transparent;
+    font-size: 0.9em;
+    white-space: nowrap;
+}
+.execute-write-analysis-allowed {
+    color: #267a3e;
+    font-weight: 700;
+}
+.execute-write-analysis-denied {
+    color: #b00020;
+    font-weight: 700;
+}
+.execute-write-analysis-na {
+    color: #687386;
+    font-style: italic;
+}
+</style>

datasette/templates/_facet_results.html

@@ -0,0 +1,28 @@
+<div class="facet-results">
+    {% for facet_info in sorted_facet_results %}
+        <div class="facet-info facet-{{ database|to_css_class }}-{{ table|to_css_class }}-{{ facet_info.name|to_css_class }}" id="facet-{{ facet_info.name|to_css_class }}" data-column="{{ facet_info.name }}">
+            <p class="facet-info-name">
+                <strong>{{ facet_info.name }}{% if facet_info.type != "column" %} ({{ facet_info.type }}){% endif %}
+                    <span class="facet-info-total">{% if facet_info.truncated %}&gt;{% endif %}{{ facet_info.results|length }}</span>
+                </strong>
+                {% if facet_info.hideable %}
+                    <a href="{{ facet_info.toggle_url }}" class="cross">&#x2716;</a>
+                {% endif %}
+            </p>
+            <ul class="tight-bullets">
+                {% for facet_value in facet_info.results %}
+                    {% if not facet_value.selected %}
+                        <li><a href="{{ facet_value.toggle_url }}" data-facet-value="{{ facet_value.value }}">{{ (facet_value.label | string()) or "-" }}</a> {{ "{:,}".format(facet_value.count) }}</li>
+                    {% else %}
+                        <li>{{ facet_value.label or "-" }} &middot; {{ "{:,}".format(facet_value.count) }} <a href="{{ facet_value.toggle_url }}" class="cross">&#x2716;</a></li>
+                    {% endif %}
+                {% endfor %}
+                {% if facet_info.truncated %}
+                    <li class="facet-truncated">{% if request.args._facet_size != "max" -%}
+                        <a href="{{ path_with_replaced_args(request, {"_facet_size": "max"}) }}">…</a>{% else -%}…{% endif %}
+                    </li>
+                {% endif %}
+            </ul>
+        </div>
+    {% endfor %}
+</div>

datasette/templates/_footer.html

@@ -1,5 +1,5 @@
-Powered by <a href="https://github.com/simonw/datasette" title="Datasette v{{ datasette_version }}">Datasette</a>
-{% if query_ms %}&middot; Query took {{ query_ms|round(3) }}ms{% endif %}
+Powered by <a href="https://datasette.io/" title="Datasette v{{ datasette_version }}">Datasette</a>
+{% if query_ms %}&middot; Queries took {{ query_ms|round(3) }}ms{% endif %}
 {% if metadata %}
     {% if metadata.license or metadata.license_url %}&middot; Data license:
         {% if metadata.license_url %}

datasette/templates/_permission_ui_styles.html

@@ -0,0 +1,145 @@
+<style>
+.permission-form {
+    background-color: #f5f5f5;
+    border: 1px solid #ddd;
+    border-radius: 5px;
+    padding: 1.5em;
+    margin-bottom: 2em;
+}
+.form-section {
+    margin-bottom: 1em;
+}
+.form-section label {
+    display: block;
+    margin-bottom: 0.3em;
+    font-weight: bold;
+}
+.form-section input[type="text"],
+.form-section select {
+    width: 100%;
+    max-width: 500px;
+    padding: 0.5em;
+    box-sizing: border-box;
+    border: 1px solid #ccc;
+    border-radius: 3px;
+}
+.form-section input[type="text"]:focus,
+.form-section select:focus {
+    outline: 2px solid #0066cc;
+    border-color: #0066cc;
+}
+.form-section small {
+    display: block;
+    margin-top: 0.3em;
+    color: #666;
+}
+.form-actions {
+    margin-top: 1em;
+}
+.submit-btn {
+    padding: 0.6em 1.5em;
+    font-size: 1em;
+    background-color: #0066cc;
+    color: white;
+    border: none;
+    border-radius: 3px;
+    cursor: pointer;
+}
+.submit-btn:hover {
+    background-color: #0052a3;
+}
+.submit-btn:disabled {
+    background-color: #ccc;
+    cursor: not-allowed;
+}
+.results-container {
+    margin-top: 2em;
+}
+.results-header {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    margin-bottom: 1em;
+}
+.results-count {
+    font-size: 0.9em;
+    color: #666;
+}
+.results-table {
+    width: 100%;
+    border-collapse: collapse;
+    background-color: white;
+    box-shadow: 0 1px 3px rgba(0,0,0,0.1);
+}
+.results-table th {
+    background-color: #f5f5f5;
+    padding: 0.75em;
+    text-align: left;
+    font-weight: bold;
+    border-bottom: 2px solid #ddd;
+}
+.results-table td {
+    padding: 0.75em;
+    border-bottom: 1px solid #eee;
+}
+.results-table tr:hover {
+    background-color: #f9f9f9;
+}
+.results-table tr.allow-row {
+    background-color: #f1f8f4;
+}
+.results-table tr.allow-row:hover {
+    background-color: #e8f5e9;
+}
+.results-table tr.deny-row {
+    background-color: #fef5f5;
+}
+.results-table tr.deny-row:hover {
+    background-color: #ffebee;
+}
+.resource-path {
+    font-family: monospace;
+    background-color: #f5f5f5;
+    padding: 0.2em 0.4em;
+    border-radius: 3px;
+}
+.pagination {
+    margin-top: 1.5em;
+    display: flex;
+    gap: 1em;
+    align-items: center;
+}
+.pagination a {
+    padding: 0.5em 1em;
+    background-color: #0066cc;
+    color: white;
+    text-decoration: none;
+    border-radius: 3px;
+}
+.pagination a:hover {
+    background-color: #0052a3;
+}
+.pagination span {
+    color: #666;
+}
+.no-results {
+    padding: 2em;
+    text-align: center;
+    color: #666;
+    background-color: #f9f9f9;
+    border: 1px solid #ddd;
+    border-radius: 5px;
+}
+.error-message {
+    padding: 1em;
+    background-color: #ffebee;
+    border: 2px solid #f44336;
+    border-radius: 5px;
+    color: #c62828;
+}
+.loading {
+    padding: 2em;
+    text-align: center;
+    color: #666;
+}
+</style>

datasette/templates/_permissions_debug_tabs.html

@@ -0,0 +1,54 @@
+{% if has_debug_permission %}
+{% set query_string = '?' + request.query_string if request.query_string else '' %}
+
+<style>
+.permissions-debug-tabs {
+    border-bottom: 2px solid #e0e0e0;
+    margin-bottom: 2em;
+    display: flex;
+    flex-wrap: wrap;
+    gap: 0.5em;
+}
+.permissions-debug-tabs a {
+    padding: 0.75em 1.25em;
+    text-decoration: none;
+    color: #333;
+    border-bottom: 3px solid transparent;
+    margin-bottom: -2px;
+    transition: all 0.2s;
+    font-weight: 500;
+}
+.permissions-debug-tabs a:hover {
+    background-color: #f5f5f5;
+    border-bottom-color: #999;
+}
+.permissions-debug-tabs a.active {
+    color: #0066cc;
+    border-bottom-color: #0066cc;
+    background-color: #f0f7ff;
+}
+@media only screen and (max-width: 576px) {
+    .permissions-debug-tabs {
+        flex-direction: column;
+        gap: 0;
+    }
+    .permissions-debug-tabs a {
+        border-bottom: 1px solid #e0e0e0;
+        margin-bottom: 0;
+    }
+    .permissions-debug-tabs a.active {
+        border-left: 3px solid #0066cc;
+        border-bottom: 1px solid #e0e0e0;
+    }
+}
+</style>
+
+<nav class="permissions-debug-tabs">
+    <a href="{{ urls.path('-/permissions') }}" {% if current_tab == "permissions" %}class="active"{% endif %}>Playground</a>
+    <a href="{{ urls.path('-/check') }}{{ query_string }}" {% if current_tab == "check" %}class="active"{% endif %}>Check</a>
+    <a href="{{ urls.path('-/allowed') }}{{ query_string }}" {% if current_tab == "allowed" %}class="active"{% endif %}>Allowed</a>
+    <a href="{{ urls.path('-/rules') }}{{ query_string }}" {% if current_tab == "rules" %}class="active"{% endif %}>Rules</a>
+    <a href="{{ urls.path('-/actions') }}" {% if current_tab == "actions" %}class="active"{% endif %}>Actions</a>
+    <a href="{{ urls.path('-/allow-debug') }}" {% if current_tab == "allow_debug" %}class="active"{% endif %}>Allow debug</a>
+</nav>
+{% endif %}

datasette/templates/_sql_parameter_scripts.html

@@ -0,0 +1,293 @@
+<script>
+window.datasetteSqlParameters = (() => {
+  if (
+    window.datasetteSqlParameters &&
+    window.datasetteSqlParameters.setupSqlParameterRefresh
+  ) {
+    return window.datasetteSqlParameters;
+  }
+
+  function currentSql(form) {
+    if (window.editor) {
+      return window.editor.state.doc.toString();
+    }
+    const sqlInput = form.querySelector("textarea#sql-editor, input[name=sql]");
+    return sqlInput ? sqlInput.value : "";
+  }
+
+  function controlState(control) {
+    return {
+      value: control.value,
+      expanded: control.tagName.toLowerCase() === "textarea",
+    };
+  }
+
+  function syncParameterState(manager) {
+    manager.parameterState = new Map();
+    manager.section
+      .querySelectorAll("[data-parameter-control]")
+      .forEach((control) => {
+        manager.parameterState.set(control.name, controlState(control));
+      });
+  }
+
+  function createControl(parameter, id, state) {
+    const control = document.createElement(state.expanded ? "textarea" : "input");
+    control.id = id;
+    control.name = parameter;
+    control.value = state.value;
+    control.setAttribute("data-parameter-control", "");
+    if (state.expanded) {
+      control.rows = 5;
+    } else {
+      control.type = "text";
+    }
+    return control;
+  }
+
+  function replaceParameterControl(
+    manager,
+    control,
+    button,
+    expand,
+    value,
+    selectionStart
+  ) {
+    const replacement = createControl(control.name, control.id, {
+      value: value === undefined ? control.value : value,
+      expanded: expand,
+    });
+    button.textContent = expand ? "Collapse" : "Expand";
+    button.setAttribute("aria-expanded", expand ? "true" : "false");
+    control.replaceWith(replacement);
+    replacement.focus();
+    if (selectionStart !== undefined && replacement.setSelectionRange) {
+      replacement.setSelectionRange(selectionStart, selectionStart);
+    }
+    manager.parameterState.set(replacement.name, controlState(replacement));
+  }
+
+  function renderParameters(manager, parameters) {
+    syncParameterState(manager);
+    const previousState = manager.parameterState;
+    const nextState = new Map();
+    manager.section.replaceChildren();
+    if (!parameters.length) {
+      manager.parameterState = nextState;
+      return;
+    }
+
+    const heading = document.createElement("h2");
+    heading.textContent = "Parameters";
+    manager.section.appendChild(heading);
+
+    parameters.forEach((parameter, index) => {
+      const id = `qp${index + 1}`;
+      const state = previousState.get(parameter) || {
+        value: "",
+        expanded: false,
+      };
+      if (!manager.allowExpand) {
+        state.expanded = false;
+      }
+      nextState.set(parameter, state);
+
+      const row = document.createElement("p");
+      row.className = "sql-parameter-row";
+
+      const label = document.createElement("label");
+      label.htmlFor = id;
+      label.textContent = parameter;
+
+      const control = createControl(parameter, id, state);
+
+      row.append(label, control);
+      if (manager.allowExpand) {
+        const button = document.createElement("button");
+        button.type = "button";
+        button.className = "sql-parameter-toggle";
+        button.setAttribute("data-parameter-toggle", "");
+        button.setAttribute("aria-controls", id);
+        button.setAttribute("aria-expanded", state.expanded ? "true" : "false");
+        button.textContent = state.expanded ? "Collapse" : "Expand";
+        row.append(" ", button);
+      }
+      manager.section.appendChild(row);
+    });
+
+    manager.parameterState = nextState;
+  }
+
+  function bindParameterControls(manager) {
+    manager.form.addEventListener("input", (event) => {
+      const control = event.target;
+      if (!control.matches || !control.matches("[data-parameter-control]")) {
+        return;
+      }
+      manager.parameterState.set(control.name, controlState(control));
+    });
+
+    if (!manager.allowExpand) {
+      return;
+    }
+
+    manager.form.addEventListener("click", (event) => {
+      const button = event.target.closest
+        ? event.target.closest("[data-parameter-toggle]")
+        : null;
+      if (!button || !manager.form.contains(button)) {
+        return;
+      }
+      const control = document.getElementById(button.getAttribute("aria-controls"));
+      if (!control) {
+        return;
+      }
+      const expanded = control.tagName.toLowerCase() === "textarea";
+      replaceParameterControl(manager, control, button, !expanded);
+    });
+
+    manager.form.addEventListener("paste", (event) => {
+      const control = event.target;
+      if (
+        !(control instanceof HTMLInputElement) ||
+        !control.matches("[data-parameter-control]")
+      ) {
+        return;
+      }
+      const pasted = event.clipboardData ? event.clipboardData.getData("text") : "";
+      if (!/[\r\n]/.test(pasted)) {
+        return;
+      }
+      const button = document.querySelector(
+        `[data-parameter-toggle][aria-controls="${control.id}"]`
+      );
+      if (!button) {
+        return;
+      }
+      event.preventDefault();
+      const selectionStart = control.selectionStart ?? control.value.length;
+      const selectionEnd = control.selectionEnd ?? selectionStart;
+      const value =
+        control.value.slice(0, selectionStart) +
+        pasted +
+        control.value.slice(selectionEnd);
+      replaceParameterControl(
+        manager,
+        control,
+        button,
+        true,
+        value,
+        selectionStart + pasted.length
+      );
+    });
+  }
+
+  function bindEditorChanges(form, callback) {
+    const editorElement = form.querySelector(".cm-content");
+    if (editorElement) {
+      editorElement.addEventListener("input", callback);
+    }
+    if (!window.editor) {
+      const sqlInput = form.querySelector("textarea#sql-editor");
+      if (sqlInput) {
+        sqlInput.addEventListener("input", callback);
+      }
+      return;
+    }
+    if (!window.editor.datasetteSqlParameterCallbacks) {
+      const editor = window.editor;
+      const originalDispatch = editor.dispatch.bind(editor);
+      editor.datasetteSqlParameterCallbacks = [];
+      editor.dispatch = (...transactions) => {
+        const before = editor.state.doc.toString();
+        originalDispatch(...transactions);
+        if (editor.state.doc.toString() !== before) {
+          editor.datasetteSqlParameterCallbacks.forEach((listener) => listener());
+        }
+      };
+    }
+    window.editor.datasetteSqlParameterCallbacks.push(callback);
+  }
+
+  function setupSqlParameterRefresh(options) {
+    const form =
+      options.form || document.querySelector("form.sql.core[data-parameters-url]");
+    if (!form) {
+      return null;
+    }
+    const shouldRenderParameters = options.renderParameters !== false;
+    const section =
+      options.section || form.querySelector("[data-sql-parameters-section]");
+    if (shouldRenderParameters && !section) {
+      return null;
+    }
+    const manager = {
+      form,
+      section,
+      allowExpand:
+        options.allowExpand === undefined
+          ? section
+            ? section.dataset.allowExpand === "1"
+            : false
+          : options.allowExpand,
+      parameterState: new Map(),
+    };
+    if (section) {
+      bindParameterControls(manager);
+      syncParameterState(manager);
+    }
+
+    const url = options.url || form.dataset.parametersUrl;
+    let refreshTimer = null;
+    let refreshSequence = 0;
+
+    async function refreshParameters() {
+      if (!url) {
+        return;
+      }
+      const sequence = ++refreshSequence;
+      try {
+        const requestUrl = new URL(url, window.location.href);
+        requestUrl.searchParams.set("sql", currentSql(form));
+        const response = await fetch(requestUrl, {
+          headers: { accept: "application/json" },
+        });
+        const data = await response.json();
+        if (sequence !== refreshSequence) {
+          return;
+        }
+        if (!response.ok) {
+          throw new Error((data.errors || [response.statusText]).join("; "));
+        }
+        if (shouldRenderParameters) {
+          renderParameters(manager, data.parameters || []);
+        }
+        if (options.onData) {
+          options.onData(data, manager);
+        }
+      } catch (error) {
+        if (sequence !== refreshSequence) {
+          return;
+        }
+        if (options.onError) {
+          options.onError(error, manager);
+        }
+      }
+    }
+
+    function scheduleRefresh() {
+      clearTimeout(refreshTimer);
+      refreshTimer = setTimeout(refreshParameters, options.debounceMs || 350);
+    }
+
+    bindEditorChanges(form, scheduleRefresh);
+    return {
+      currentSql: () => currentSql(form),
+      refreshParameters,
+      renderParameters: (parameters) => renderParameters(manager, parameters),
+    };
+  }
+
+  return { setupSqlParameterRefresh };
+})();
+</script>

datasette/templates/_sql_parameter_styles.html

@@ -0,0 +1,58 @@
+<style>
+form.sql .sql-editor {
+    max-width: 52rem;
+}
+form.sql .sql-editor textarea#sql-editor {
+    width: 100%;
+}
+form.sql .sql-parameters-section {
+    max-width: 52rem;
+}
+form.sql .sql-parameter-row {
+    align-items: start;
+    column-gap: 0.6rem;
+    display: grid;
+    grid-template-columns: minmax(8rem, 11rem) minmax(16rem, 1fr) auto;
+    margin: 0 0 0.65rem;
+    max-width: 52rem;
+}
+form.sql .sql-parameter-row label {
+    overflow-wrap: anywhere;
+    padding-top: 0.55rem;
+    width: auto;
+}
+form.sql .sql-parameter-row input[data-parameter-control],
+form.sql .sql-parameter-row textarea[data-parameter-control] {
+    box-sizing: border-box;
+    width: 100%;
+}
+form.sql .sql-parameter-row textarea[data-parameter-control] {
+    border: 1px solid #ccc;
+    border-radius: 3px;
+    display: block;
+    font-family: Helvetica, sans-serif;
+    font-size: 1em;
+    min-height: 7rem;
+    padding: 9px 4px;
+}
+form.sql.core button.sql-parameter-toggle[type=button] {
+    font-size: 0.72rem;
+    height: 1.8rem;
+    line-height: 1;
+    margin: 0.25rem 0 0;
+    padding: 0.25rem 0.45rem;
+}
+@media (max-width: 480px) {
+    form.sql .sql-parameter-row {
+        grid-template-columns: 1fr;
+        row-gap: 0.25rem;
+    }
+    form.sql .sql-parameter-row label {
+        padding-top: 0;
+    }
+    form.sql.core button.sql-parameter-toggle[type=button] {
+        justify-self: start;
+        margin-top: 0;
+    }
+}
+</style>

datasette/templates/_sql_parameters.html

@@ -0,0 +1,9 @@
+<div id="{{ sql_parameters_section_id|default("sql-parameters-section") }}" class="sql-parameters-section" data-sql-parameters-section{% if sql_parameters_allow_expand|default(false) %} data-allow-expand="1"{% endif %}>
+    {% if parameter_names %}
+        <h2>Parameters</h2>
+        {% for parameter in parameter_names %}
+            {% set parameter_id = (sql_parameter_id_prefix|default("qp")) ~ loop.index %}
+            <p class="sql-parameter-row"><label for="{{ parameter_id }}">{{ parameter }}</label> <input type="text" id="{{ parameter_id }}" name="{{ parameter }}" value="{{ parameter_values.get(parameter, "") }}" data-parameter-control>{% if sql_parameters_allow_expand|default(false) %} <button type="button" class="sql-parameter-toggle" data-parameter-toggle aria-controls="{{ parameter_id }}" aria-expanded="false">Expand</button>{% endif %}</p>
+        {% endfor %}
+    {% endif %}
+</div>